kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/mutex.h>
  11 #include <linux/slab.h>
  12 #include <linux/stacktrace.h>
  13 #include <linux/rculist.h>
  14 #include <linux/tracefs.h>
  15
  16 #include "tracing_map.h"
  17 #include "trace.h"
  18
  19 #define SYNTH_SYSTEM            "synthetic"
  20 #define SYNTH_FIELDS_MAX        16
  21
  22 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  23
  24 struct hist_field;
  25
  26 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  27                                 struct tracing_map_elt *elt,
  28                                 struct ring_buffer_event *rbe,
  29                                 void *event);
  30
  31 #define HIST_FIELD_OPERANDS_MAX 2
  32 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  33 #define HIST_ACTIONS_MAX        8
  34
  35 enum field_op_id {
  36         FIELD_OP_NONE,
  37         FIELD_OP_PLUS,
  38         FIELD_OP_MINUS,
  39         FIELD_OP_UNARY_MINUS,
  40 };
  41
  42 struct hist_var {
  43         char                            *name;
  44         struct hist_trigger_data        *hist_data;
  45         unsigned int                    idx;
  46 };
  47
  48 struct hist_field {
  49         struct ftrace_event_field       *field;
  50         unsigned long                   flags;
  51         hist_field_fn_t                 fn;
  52         unsigned int                    size;
  53         unsigned int                    offset;
  54         unsigned int                    is_signed;
  55         const char                      *type;
  56         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
  57         struct hist_trigger_data        *hist_data;
  58         struct hist_var                 var;
  59         enum field_op_id                operator;
  60         char                            *system;
  61         char                            *event_name;
  62         char                            *name;
  63         unsigned int                    var_idx;
  64         unsigned int                    var_ref_idx;
  65         bool                            read_once;
  66 };
  67
  68 static u64 hist_field_none(struct hist_field *field,
  69                            struct tracing_map_elt *elt,
  70                            struct ring_buffer_event *rbe,
  71                            void *event)
  72 {
  73         return 0;
  74 }
  75
  76 static u64 hist_field_counter(struct hist_field *field,
  77                               struct tracing_map_elt *elt,
  78                               struct ring_buffer_event *rbe,
  79                               void *event)
  80 {
  81         return 1;
  82 }
  83
  84 static u64 hist_field_string(struct hist_field *hist_field,
  85                              struct tracing_map_elt *elt,
  86                              struct ring_buffer_event *rbe,
  87                              void *event)
  88 {
  89         char *addr = (char *)(event + hist_field->field->offset);
  90
  91         return (u64)(unsigned long)addr;
  92 }
  93
  94 static u64 hist_field_dynstring(struct hist_field *hist_field,
  95                                 struct tracing_map_elt *elt,
  96                                 struct ring_buffer_event *rbe,
  97                                 void *event)
  98 {
  99         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 100         int str_loc = str_item & 0xffff;
 101         char *addr = (char *)(event + str_loc);
 102
 103         return (u64)(unsigned long)addr;
 104 }
 105
 106 static u64 hist_field_pstring(struct hist_field *hist_field,
 107                               struct tracing_map_elt *elt,
 108                               struct ring_buffer_event *rbe,
 109                               void *event)
 110 {
 111         char **addr = (char **)(event + hist_field->field->offset);
 112
 113         return (u64)(unsigned long)*addr;
 114 }
 115
 116 static u64 hist_field_log2(struct hist_field *hist_field,
 117                            struct tracing_map_elt *elt,
 118                            struct ring_buffer_event *rbe,
 119                            void *event)
 120 {
 121         struct hist_field *operand = hist_field->operands[0];
 122
 123         u64 val = operand->fn(operand, elt, rbe, event);
 124
 125         return (u64) ilog2(roundup_pow_of_two(val));
 126 }
 127
 128 static u64 hist_field_plus(struct hist_field *hist_field,
 129                            struct tracing_map_elt *elt,
 130                            struct ring_buffer_event *rbe,
 131                            void *event)
 132 {
 133         struct hist_field *operand1 = hist_field->operands[0];
 134         struct hist_field *operand2 = hist_field->operands[1];
 135
 136         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 137         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 138
 139         return val1 + val2;
 140 }
 141
 142 static u64 hist_field_minus(struct hist_field *hist_field,
 143                             struct tracing_map_elt *elt,
 144                             struct ring_buffer_event *rbe,
 145                             void *event)
 146 {
 147         struct hist_field *operand1 = hist_field->operands[0];
 148         struct hist_field *operand2 = hist_field->operands[1];
 149
 150         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 151         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 152
 153         return val1 - val2;
 154 }
 155
 156 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 157                                   struct tracing_map_elt *elt,
 158                                   struct ring_buffer_event *rbe,
 159                                   void *event)
 160 {
 161         struct hist_field *operand = hist_field->operands[0];
 162
 163         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 164         u64 val = (u64)-sval;
 165
 166         return val;
 167 }
 168
 169 #define DEFINE_HIST_FIELD_FN(type)                                      \
 170         static u64 hist_field_##type(struct hist_field *hist_field,     \
 171                                      struct tracing_map_elt *elt,       \
 172                                      struct ring_buffer_event *rbe,     \
 173                                      void *event)                       \
 174 {                                                                       \
 175         type *addr = (type *)(event + hist_field->field->offset);       \
 176                                                                         \
 177         return (u64)(unsigned long)*addr;                               \
 178 }
 179
 180 DEFINE_HIST_FIELD_FN(s64);
 181 DEFINE_HIST_FIELD_FN(u64);
 182 DEFINE_HIST_FIELD_FN(s32);
 183 DEFINE_HIST_FIELD_FN(u32);
 184 DEFINE_HIST_FIELD_FN(s16);
 185 DEFINE_HIST_FIELD_FN(u16);
 186 DEFINE_HIST_FIELD_FN(s8);
 187 DEFINE_HIST_FIELD_FN(u8);
 188
 189 #define for_each_hist_field(i, hist_data)       \
 190         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 191
 192 #define for_each_hist_val_field(i, hist_data)   \
 193         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 194
 195 #define for_each_hist_key_field(i, hist_data)   \
 196         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 197
 198 #define HIST_STACKTRACE_DEPTH   16
 199 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 200 #define HIST_STACKTRACE_SKIP    5
 201
 202 #define HITCOUNT_IDX            0
 203 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 204
 205 enum hist_field_flags {
 206         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 207         HIST_FIELD_FL_KEY               = 1 << 1,
 208         HIST_FIELD_FL_STRING            = 1 << 2,
 209         HIST_FIELD_FL_HEX               = 1 << 3,
 210         HIST_FIELD_FL_SYM               = 1 << 4,
 211         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 212         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 213         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 214         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 215         HIST_FIELD_FL_LOG2              = 1 << 9,
 216         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 217         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 218         HIST_FIELD_FL_VAR               = 1 << 12,
 219         HIST_FIELD_FL_EXPR              = 1 << 13,
 220         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 221         HIST_FIELD_FL_CPU               = 1 << 15,
 222         HIST_FIELD_FL_ALIAS             = 1 << 16,
 223 };
 224
 225 struct var_defs {
 226         unsigned int    n_vars;
 227         char            *name[TRACING_MAP_VARS_MAX];
 228         char            *expr[TRACING_MAP_VARS_MAX];
 229 };
 230
 231 struct hist_trigger_attrs {
 232         char            *keys_str;
 233         char            *vals_str;
 234         char            *sort_key_str;
 235         char            *name;
 236         char            *clock;
 237         bool            pause;
 238         bool            cont;
 239         bool            clear;
 240         bool            ts_in_usecs;
 241         unsigned int    map_bits;
 242
 243         char            *assignment_str[TRACING_MAP_VARS_MAX];
 244         unsigned int    n_assignments;
 245
 246         char            *action_str[HIST_ACTIONS_MAX];
 247         unsigned int    n_actions;
 248
 249         struct var_defs var_defs;
 250 };
 251
 252 struct field_var {
 253         struct hist_field       *var;
 254         struct hist_field       *val;
 255 };
 256
 257 struct field_var_hist {
 258         struct hist_trigger_data        *hist_data;
 259         char                            *cmd;
 260 };
 261
 262 struct hist_trigger_data {
 263         struct hist_field               *fields[HIST_FIELDS_MAX];
 264         unsigned int                    n_vals;
 265         unsigned int                    n_keys;
 266         unsigned int                    n_fields;
 267         unsigned int                    n_vars;
 268         unsigned int                    key_size;
 269         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 270         unsigned int                    n_sort_keys;
 271         struct trace_event_file         *event_file;
 272         struct hist_trigger_attrs       *attrs;
 273         struct tracing_map              *map;
 274         bool                            enable_timestamps;
 275         bool                            remove;
 276         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 277         unsigned int                    n_var_refs;
 278
 279         struct action_data              *actions[HIST_ACTIONS_MAX];
 280         unsigned int                    n_actions;
 281
 282         struct hist_field               *synth_var_refs[SYNTH_FIELDS_MAX];
 283         unsigned int                    n_synth_var_refs;
 284         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 285         unsigned int                    n_field_vars;
 286         unsigned int                    n_field_var_str;
 287         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 288         unsigned int                    n_field_var_hists;
 289
 290         struct field_var                *max_vars[SYNTH_FIELDS_MAX];
 291         unsigned int                    n_max_vars;
 292         unsigned int                    n_max_var_str;
 293 };
 294
 295 struct synth_field {
 296         char *type;
 297         char *name;
 298         size_t size;
 299         bool is_signed;
 300         bool is_string;
 301 };
 302
 303 struct synth_event {
 304         struct list_head                        list;
 305         int                                     ref;
 306         char                                    *name;
 307         struct synth_field                      **fields;
 308         unsigned int                            n_fields;
 309         unsigned int                            n_u64;
 310         struct trace_event_class                class;
 311         struct trace_event_call                 call;
 312         struct tracepoint                       *tp;
 313 };
 314
 315 struct action_data;
 316
 317 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 318                              struct tracing_map_elt *elt, void *rec,
 319                              struct ring_buffer_event *rbe,
 320                              struct action_data *data, u64 *var_ref_vals);
 321
 322 struct action_data {
 323         action_fn_t             fn;
 324         unsigned int            n_params;
 325         char                    *params[SYNTH_FIELDS_MAX];
 326
 327         union {
 328                 struct {
 329                         unsigned int            var_ref_idx;
 330                         char                    *match_event;
 331                         char                    *match_event_system;
 332                         char                    *synth_event_name;
 333                         struct synth_event      *synth_event;
 334                 } onmatch;
 335
 336                 struct {
 337                         char                    *var_str;
 338                         char                    *fn_name;
 339                         unsigned int            max_var_ref_idx;
 340                         struct hist_field       *max_var;
 341                         struct hist_field       *var;
 342                 } onmax;
 343         };
 344 };
 345
 346
 347 static char last_hist_cmd[MAX_FILTER_STR_VAL];
 348 static char hist_err_str[MAX_FILTER_STR_VAL];
 349
 350 static void last_cmd_set(char *str)
 351 {
 352         if (!str)
 353                 return;
 354
 355         strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1);
 356 }
 357
 358 static void hist_err(char *str, char *var)
 359 {
 360         int maxlen = MAX_FILTER_STR_VAL - 1;
 361
 362         if (!str)
 363                 return;
 364
 365         if (strlen(hist_err_str))
 366                 return;
 367
 368         if (!var)
 369                 var = "";
 370
 371         if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen)
 372                 return;
 373
 374         strcat(hist_err_str, str);
 375         strcat(hist_err_str, var);
 376 }
 377
 378 static void hist_err_event(char *str, char *system, char *event, char *var)
 379 {
 380         char err[MAX_FILTER_STR_VAL];
 381
 382         if (system && var)
 383                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var);
 384         else if (system)
 385                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event);
 386         else
 387                 strscpy(err, var, MAX_FILTER_STR_VAL);
 388
 389         hist_err(str, err);
 390 }
 391
 392 static void hist_err_clear(void)
 393 {
 394         hist_err_str[0] = '\0';
 395 }
 396
 397 static bool have_hist_err(void)
 398 {
 399         if (strlen(hist_err_str))
 400                 return true;
 401
 402         return false;
 403 }
 404
 405 static LIST_HEAD(synth_event_list);
 406 static DEFINE_MUTEX(synth_event_mutex);
 407
 408 struct synth_trace_event {
 409         struct trace_entry      ent;
 410         u64                     fields[];
 411 };
 412
 413 static int synth_event_define_fields(struct trace_event_call *call)
 414 {
 415         struct synth_trace_event trace;
 416         int offset = offsetof(typeof(trace), fields);
 417         struct synth_event *event = call->data;
 418         unsigned int i, size, n_u64;
 419         char *name, *type;
 420         bool is_signed;
 421         int ret = 0;
 422
 423         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 424                 size = event->fields[i]->size;
 425                 is_signed = event->fields[i]->is_signed;
 426                 type = event->fields[i]->type;
 427                 name = event->fields[i]->name;
 428                 ret = trace_define_field(call, type, name, offset, size,
 429                                          is_signed, FILTER_OTHER);
 430                 if (ret)
 431                         break;
 432
 433                 if (event->fields[i]->is_string) {
 434                         offset += STR_VAR_LEN_MAX;
 435                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 436                 } else {
 437                         offset += sizeof(u64);
 438                         n_u64++;
 439                 }
 440         }
 441
 442         event->n_u64 = n_u64;
 443
 444         return ret;
 445 }
 446
 447 static bool synth_field_signed(char *type)
 448 {
 449         if (strncmp(type, "u", 1) == 0)
 450                 return false;
 451
 452         return true;
 453 }
 454
 455 static int synth_field_is_string(char *type)
 456 {
 457         if (strstr(type, "char[") != NULL)
 458                 return true;
 459
 460         return false;
 461 }
 462
 463 static int synth_field_string_size(char *type)
 464 {
 465         char buf[4], *end, *start;
 466         unsigned int len;
 467         int size, err;
 468
 469         start = strstr(type, "char[");
 470         if (start == NULL)
 471                 return -EINVAL;
 472         start += strlen("char[");
 473
 474         end = strchr(type, ']');
 475         if (!end || end < start)
 476                 return -EINVAL;
 477
 478         len = end - start;
 479         if (len > 3)
 480                 return -EINVAL;
 481
 482         strncpy(buf, start, len);
 483         buf[len] = '\0';
 484
 485         err = kstrtouint(buf, 0, &size);
 486         if (err)
 487                 return err;
 488
 489         if (size > STR_VAR_LEN_MAX)
 490                 return -EINVAL;
 491
 492         return size;
 493 }
 494
 495 static int synth_field_size(char *type)
 496 {
 497         int size = 0;
 498
 499         if (strcmp(type, "s64") == 0)
 500                 size = sizeof(s64);
 501         else if (strcmp(type, "u64") == 0)
 502                 size = sizeof(u64);
 503         else if (strcmp(type, "s32") == 0)
 504                 size = sizeof(s32);
 505         else if (strcmp(type, "u32") == 0)
 506                 size = sizeof(u32);
 507         else if (strcmp(type, "s16") == 0)
 508                 size = sizeof(s16);
 509         else if (strcmp(type, "u16") == 0)
 510                 size = sizeof(u16);
 511         else if (strcmp(type, "s8") == 0)
 512                 size = sizeof(s8);
 513         else if (strcmp(type, "u8") == 0)
 514                 size = sizeof(u8);
 515         else if (strcmp(type, "char") == 0)
 516                 size = sizeof(char);
 517         else if (strcmp(type, "unsigned char") == 0)
 518                 size = sizeof(unsigned char);
 519         else if (strcmp(type, "int") == 0)
 520                 size = sizeof(int);
 521         else if (strcmp(type, "unsigned int") == 0)
 522                 size = sizeof(unsigned int);
 523         else if (strcmp(type, "long") == 0)
 524                 size = sizeof(long);
 525         else if (strcmp(type, "unsigned long") == 0)
 526                 size = sizeof(unsigned long);
 527         else if (strcmp(type, "pid_t") == 0)
 528                 size = sizeof(pid_t);
 529         else if (synth_field_is_string(type))
 530                 size = synth_field_string_size(type);
 531
 532         return size;
 533 }
 534
 535 static const char *synth_field_fmt(char *type)
 536 {
 537         const char *fmt = "%llu";
 538
 539         if (strcmp(type, "s64") == 0)
 540                 fmt = "%lld";
 541         else if (strcmp(type, "u64") == 0)
 542                 fmt = "%llu";
 543         else if (strcmp(type, "s32") == 0)
 544                 fmt = "%d";
 545         else if (strcmp(type, "u32") == 0)
 546                 fmt = "%u";
 547         else if (strcmp(type, "s16") == 0)
 548                 fmt = "%d";
 549         else if (strcmp(type, "u16") == 0)
 550                 fmt = "%u";
 551         else if (strcmp(type, "s8") == 0)
 552                 fmt = "%d";
 553         else if (strcmp(type, "u8") == 0)
 554                 fmt = "%u";
 555         else if (strcmp(type, "char") == 0)
 556                 fmt = "%d";
 557         else if (strcmp(type, "unsigned char") == 0)
 558                 fmt = "%u";
 559         else if (strcmp(type, "int") == 0)
 560                 fmt = "%d";
 561         else if (strcmp(type, "unsigned int") == 0)
 562                 fmt = "%u";
 563         else if (strcmp(type, "long") == 0)
 564                 fmt = "%ld";
 565         else if (strcmp(type, "unsigned long") == 0)
 566                 fmt = "%lu";
 567         else if (strcmp(type, "pid_t") == 0)
 568                 fmt = "%d";
 569         else if (synth_field_is_string(type))
 570                 fmt = "%s";
 571
 572         return fmt;
 573 }
 574
 575 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 576                                            int flags,
 577                                            struct trace_event *event)
 578 {
 579         struct trace_array *tr = iter->tr;
 580         struct trace_seq *s = &iter->seq;
 581         struct synth_trace_event *entry;
 582         struct synth_event *se;
 583         unsigned int i, n_u64;
 584         char print_fmt[32];
 585         const char *fmt;
 586
 587         entry = (struct synth_trace_event *)iter->ent;
 588         se = container_of(event, struct synth_event, call.event);
 589
 590         trace_seq_printf(s, "%s: ", se->name);
 591
 592         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 593                 if (trace_seq_has_overflowed(s))
 594                         goto end;
 595
 596                 fmt = synth_field_fmt(se->fields[i]->type);
 597
 598                 /* parameter types */
 599                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
 600                         trace_seq_printf(s, "%s ", fmt);
 601
 602                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 603
 604                 /* parameter values */
 605                 if (se->fields[i]->is_string) {
 606                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 607                                          (char *)&entry->fields[n_u64],
 608                                          i == se->n_fields - 1 ? "" : " ");
 609                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 610                 } else {
 611                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 612                                          entry->fields[n_u64],
 613                                          i == se->n_fields - 1 ? "" : " ");
 614                         n_u64++;
 615                 }
 616         }
 617 end:
 618         trace_seq_putc(s, '\n');
 619
 620         return trace_handle_return(s);
 621 }
 622
 623 static struct trace_event_functions synth_event_funcs = {
 624         .trace          = print_synth_event
 625 };
 626
 627 static notrace void trace_event_raw_event_synth(void *__data,
 628                                                 u64 *var_ref_vals,
 629                                                 unsigned int var_ref_idx)
 630 {
 631         struct trace_event_file *trace_file = __data;
 632         struct synth_trace_event *entry;
 633         struct trace_event_buffer fbuffer;
 634         struct ring_buffer *buffer;
 635         struct synth_event *event;
 636         unsigned int i, n_u64;
 637         int fields_size = 0;
 638
 639         event = trace_file->event_call->data;
 640
 641         if (trace_trigger_soft_disabled(trace_file))
 642                 return;
 643
 644         fields_size = event->n_u64 * sizeof(u64);
 645
 646         /*
 647          * Avoid ring buffer recursion detection, as this event
 648          * is being performed within another event.
 649          */
 650         buffer = trace_file->tr->trace_buffer.buffer;
 651         ring_buffer_nest_start(buffer);
 652
 653         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 654                                            sizeof(*entry) + fields_size);
 655         if (!entry)
 656                 goto out;
 657
 658         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 659                 if (event->fields[i]->is_string) {
 660                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 661                         char *str_field = (char *)&entry->fields[n_u64];
 662
 663                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 664                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 665                 } else {
 666                         entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
 667                         n_u64++;
 668                 }
 669         }
 670
 671         trace_event_buffer_commit(&fbuffer);
 672 out:
 673         ring_buffer_nest_end(buffer);
 674 }
 675
 676 static void free_synth_event_print_fmt(struct trace_event_call *call)
 677 {
 678         if (call) {
 679                 kfree(call->print_fmt);
 680                 call->print_fmt = NULL;
 681         }
 682 }
 683
 684 static int __set_synth_event_print_fmt(struct synth_event *event,
 685                                        char *buf, int len)
 686 {
 687         const char *fmt;
 688         int pos = 0;
 689         int i;
 690
 691         /* When len=0, we just calculate the needed length */
 692 #define LEN_OR_ZERO (len ? len - pos : 0)
 693
 694         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 695         for (i = 0; i < event->n_fields; i++) {
 696                 fmt = synth_field_fmt(event->fields[i]->type);
 697                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 698                                 event->fields[i]->name, fmt,
 699                                 i == event->n_fields - 1 ? "" : ", ");
 700         }
 701         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 702
 703         for (i = 0; i < event->n_fields; i++) {
 704                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 705                                 ", REC->%s", event->fields[i]->name);
 706         }
 707
 708 #undef LEN_OR_ZERO
 709
 710         /* return the length of print_fmt */
 711         return pos;
 712 }
 713
 714 static int set_synth_event_print_fmt(struct trace_event_call *call)
 715 {
 716         struct synth_event *event = call->data;
 717         char *print_fmt;
 718         int len;
 719
 720         /* First: called with 0 length to calculate the needed length */
 721         len = __set_synth_event_print_fmt(event, NULL, 0);
 722
 723         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 724         if (!print_fmt)
 725                 return -ENOMEM;
 726
 727         /* Second: actually write the @print_fmt */
 728         __set_synth_event_print_fmt(event, print_fmt, len + 1);
 729         call->print_fmt = print_fmt;
 730
 731         return 0;
 732 }
 733
 734 static void free_synth_field(struct synth_field *field)
 735 {
 736         kfree(field->type);
 737         kfree(field->name);
 738         kfree(field);
 739 }
 740
 741 static struct synth_field *parse_synth_field(char *field_type,
 742                                              char *field_name)
 743 {
 744         struct synth_field *field;
 745         int len, ret = 0;
 746         char *array;
 747
 748         if (field_type[0] == ';')
 749                 field_type++;
 750
 751         len = strlen(field_name);
 752         if (field_name[len - 1] == ';')
 753                 field_name[len - 1] = '\0';
 754
 755         field = kzalloc(sizeof(*field), GFP_KERNEL);
 756         if (!field)
 757                 return ERR_PTR(-ENOMEM);
 758
 759         len = strlen(field_type) + 1;
 760         array = strchr(field_name, '[');
 761         if (array)
 762                 len += strlen(array);
 763         field->type = kzalloc(len, GFP_KERNEL);
 764         if (!field->type) {
 765                 ret = -ENOMEM;
 766                 goto free;
 767         }
 768         strcat(field->type, field_type);
 769         if (array) {
 770                 strcat(field->type, array);
 771                 *array = '\0';
 772         }
 773
 774         field->size = synth_field_size(field->type);
 775         if (!field->size) {
 776                 ret = -EINVAL;
 777                 goto free;
 778         }
 779
 780         if (synth_field_is_string(field->type))
 781                 field->is_string = true;
 782
 783         field->is_signed = synth_field_signed(field->type);
 784
 785         field->name = kstrdup(field_name, GFP_KERNEL);
 786         if (!field->name) {
 787                 ret = -ENOMEM;
 788                 goto free;
 789         }
 790  out:
 791         return field;
 792  free:
 793         free_synth_field(field);
 794         field = ERR_PTR(ret);
 795         goto out;
 796 }
 797
 798 static void free_synth_tracepoint(struct tracepoint *tp)
 799 {
 800         if (!tp)
 801                 return;
 802
 803         kfree(tp->name);
 804         kfree(tp);
 805 }
 806
 807 static struct tracepoint *alloc_synth_tracepoint(char *name)
 808 {
 809         struct tracepoint *tp;
 810
 811         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
 812         if (!tp)
 813                 return ERR_PTR(-ENOMEM);
 814
 815         tp->name = kstrdup(name, GFP_KERNEL);
 816         if (!tp->name) {
 817                 kfree(tp);
 818                 return ERR_PTR(-ENOMEM);
 819         }
 820
 821         return tp;
 822 }
 823
 824 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 825                                     unsigned int var_ref_idx);
 826
 827 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 828                                unsigned int var_ref_idx)
 829 {
 830         struct tracepoint *tp = event->tp;
 831
 832         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 833                 struct tracepoint_func *probe_func_ptr;
 834                 synth_probe_func_t probe_func;
 835                 void *__data;
 836
 837                 if (!(cpu_online(raw_smp_processor_id())))
 838                         return;
 839
 840                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 841                 if (probe_func_ptr) {
 842                         do {
 843                                 probe_func = probe_func_ptr->func;
 844                                 __data = probe_func_ptr->data;
 845                                 probe_func(__data, var_ref_vals, var_ref_idx);
 846                         } while ((++probe_func_ptr)->func);
 847                 }
 848         }
 849 }
 850
 851 static struct synth_event *find_synth_event(const char *name)
 852 {
 853         struct synth_event *event;
 854
 855         list_for_each_entry(event, &synth_event_list, list) {
 856                 if (strcmp(event->name, name) == 0)
 857                         return event;
 858         }
 859
 860         return NULL;
 861 }
 862
 863 static int register_synth_event(struct synth_event *event)
 864 {
 865         struct trace_event_call *call = &event->call;
 866         int ret = 0;
 867
 868         event->call.class = &event->class;
 869         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
 870         if (!event->class.system) {
 871                 ret = -ENOMEM;
 872                 goto out;
 873         }
 874
 875         event->tp = alloc_synth_tracepoint(event->name);
 876         if (IS_ERR(event->tp)) {
 877                 ret = PTR_ERR(event->tp);
 878                 event->tp = NULL;
 879                 goto out;
 880         }
 881
 882         INIT_LIST_HEAD(&call->class->fields);
 883         call->event.funcs = &synth_event_funcs;
 884         call->class->define_fields = synth_event_define_fields;
 885
 886         ret = register_trace_event(&call->event);
 887         if (!ret) {
 888                 ret = -ENODEV;
 889                 goto out;
 890         }
 891         call->flags = TRACE_EVENT_FL_TRACEPOINT;
 892         call->class->reg = trace_event_reg;
 893         call->class->probe = trace_event_raw_event_synth;
 894         call->data = event;
 895         call->tp = event->tp;
 896
 897         ret = trace_add_event_call(call);
 898         if (ret) {
 899                 pr_warn("Failed to register synthetic event: %s\n",
 900                         trace_event_name(call));
 901                 goto err;
 902         }
 903
 904         ret = set_synth_event_print_fmt(call);
 905         if (ret < 0) {
 906                 trace_remove_event_call(call);
 907                 goto err;
 908         }
 909  out:
 910         return ret;
 911  err:
 912         unregister_trace_event(&call->event);
 913         goto out;
 914 }
 915
 916 static int unregister_synth_event(struct synth_event *event)
 917 {
 918         struct trace_event_call *call = &event->call;
 919         int ret;
 920
 921         ret = trace_remove_event_call(call);
 922
 923         return ret;
 924 }
 925
 926 static void free_synth_event(struct synth_event *event)
 927 {
 928         unsigned int i;
 929
 930         if (!event)
 931                 return;
 932
 933         for (i = 0; i < event->n_fields; i++)
 934                 free_synth_field(event->fields[i]);
 935
 936         kfree(event->fields);
 937         kfree(event->name);
 938         kfree(event->class.system);
 939         free_synth_tracepoint(event->tp);
 940         free_synth_event_print_fmt(&event->call);
 941         kfree(event);
 942 }
 943
 944 static struct synth_event *alloc_synth_event(char *event_name, int n_fields,
 945                                              struct synth_field **fields)
 946 {
 947         struct synth_event *event;
 948         unsigned int i;
 949
 950         event = kzalloc(sizeof(*event), GFP_KERNEL);
 951         if (!event) {
 952                 event = ERR_PTR(-ENOMEM);
 953                 goto out;
 954         }
 955
 956         event->name = kstrdup(event_name, GFP_KERNEL);
 957         if (!event->name) {
 958                 kfree(event);
 959                 event = ERR_PTR(-ENOMEM);
 960                 goto out;
 961         }
 962
 963         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
 964         if (!event->fields) {
 965                 free_synth_event(event);
 966                 event = ERR_PTR(-ENOMEM);
 967                 goto out;
 968         }
 969
 970         for (i = 0; i < n_fields; i++)
 971                 event->fields[i] = fields[i];
 972
 973         event->n_fields = n_fields;
 974  out:
 975         return event;
 976 }
 977
 978 static void action_trace(struct hist_trigger_data *hist_data,
 979                          struct tracing_map_elt *elt, void *rec,
 980                          struct ring_buffer_event *rbe,
 981                          struct action_data *data, u64 *var_ref_vals)
 982 {
 983         struct synth_event *event = data->onmatch.synth_event;
 984
 985         trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx);
 986 }
 987
 988 struct hist_var_data {
 989         struct list_head list;
 990         struct hist_trigger_data *hist_data;
 991 };
 992
 993 static void add_or_delete_synth_event(struct synth_event *event, int delete)
 994 {
 995         if (delete)
 996                 free_synth_event(event);
 997         else {
 998                 mutex_lock(&synth_event_mutex);
 999                 if (!find_synth_event(event->name))
1000                         list_add(&event->list, &synth_event_list);
1001                 else
1002                         free_synth_event(event);
1003                 mutex_unlock(&synth_event_mutex);
1004         }
1005 }
1006
1007 static int create_synth_event(int argc, char **argv)
1008 {
1009         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1010         struct synth_event *event = NULL;
1011         bool delete_event = false;
1012         int i, n_fields = 0, ret = 0;
1013         char *name;
1014
1015         mutex_lock(&synth_event_mutex);
1016
1017         /*
1018          * Argument syntax:
1019          *  - Add synthetic event: <event_name> field[;field] ...
1020          *  - Remove synthetic event: !<event_name> field[;field] ...
1021          *      where 'field' = type field_name
1022          */
1023         if (argc < 1) {
1024                 ret = -EINVAL;
1025                 goto out;
1026         }
1027
1028         name = argv[0];
1029         if (name[0] == '!') {
1030                 delete_event = true;
1031                 name++;
1032         }
1033
1034         event = find_synth_event(name);
1035         if (event) {
1036                 if (delete_event) {
1037                         if (event->ref) {
1038                                 event = NULL;
1039                                 ret = -EBUSY;
1040                                 goto out;
1041                         }
1042                         list_del(&event->list);
1043                         goto out;
1044                 }
1045                 event = NULL;
1046                 ret = -EEXIST;
1047                 goto out;
1048         } else if (delete_event)
1049                 goto out;
1050
1051         if (argc < 2) {
1052                 ret = -EINVAL;
1053                 goto out;
1054         }
1055
1056         for (i = 1; i < argc - 1; i++) {
1057                 if (strcmp(argv[i], ";") == 0)
1058                         continue;
1059                 if (n_fields == SYNTH_FIELDS_MAX) {
1060                         ret = -EINVAL;
1061                         goto err;
1062                 }
1063
1064                 field = parse_synth_field(argv[i], argv[i + 1]);
1065                 if (IS_ERR(field)) {
1066                         ret = PTR_ERR(field);
1067                         goto err;
1068                 }
1069                 fields[n_fields] = field;
1070                 i++; n_fields++;
1071         }
1072
1073         if (i < argc) {
1074                 ret = -EINVAL;
1075                 goto err;
1076         }
1077
1078         event = alloc_synth_event(name, n_fields, fields);
1079         if (IS_ERR(event)) {
1080                 ret = PTR_ERR(event);
1081                 event = NULL;
1082                 goto err;
1083         }
1084  out:
1085         mutex_unlock(&synth_event_mutex);
1086
1087         if (event) {
1088                 if (delete_event) {
1089                         ret = unregister_synth_event(event);
1090                         add_or_delete_synth_event(event, !ret);
1091                 } else {
1092                         ret = register_synth_event(event);
1093                         add_or_delete_synth_event(event, ret);
1094                 }
1095         }
1096
1097         return ret;
1098  err:
1099         mutex_unlock(&synth_event_mutex);
1100
1101         for (i = 0; i < n_fields; i++)
1102                 free_synth_field(fields[i]);
1103         free_synth_event(event);
1104
1105         return ret;
1106 }
1107
1108 static int release_all_synth_events(void)
1109 {
1110         struct list_head release_events;
1111         struct synth_event *event, *e;
1112         int ret = 0;
1113
1114         INIT_LIST_HEAD(&release_events);
1115
1116         mutex_lock(&synth_event_mutex);
1117
1118         list_for_each_entry(event, &synth_event_list, list) {
1119                 if (event->ref) {
1120                         mutex_unlock(&synth_event_mutex);
1121                         return -EBUSY;
1122                 }
1123         }
1124
1125         list_splice_init(&event->list, &release_events);
1126
1127         mutex_unlock(&synth_event_mutex);
1128
1129         list_for_each_entry_safe(event, e, &release_events, list) {
1130                 list_del(&event->list);
1131
1132                 ret = unregister_synth_event(event);
1133                 add_or_delete_synth_event(event, !ret);
1134         }
1135
1136         return ret;
1137 }
1138
1139
1140 static void *synth_events_seq_start(struct seq_file *m, loff_t *pos)
1141 {
1142         mutex_lock(&synth_event_mutex);
1143
1144         return seq_list_start(&synth_event_list, *pos);
1145 }
1146
1147 static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos)
1148 {
1149         return seq_list_next(v, &synth_event_list, pos);
1150 }
1151
1152 static void synth_events_seq_stop(struct seq_file *m, void *v)
1153 {
1154         mutex_unlock(&synth_event_mutex);
1155 }
1156
1157 static int synth_events_seq_show(struct seq_file *m, void *v)
1158 {
1159         struct synth_field *field;
1160         struct synth_event *event = v;
1161         unsigned int i;
1162
1163         seq_printf(m, "%s\t", event->name);
1164
1165         for (i = 0; i < event->n_fields; i++) {
1166                 field = event->fields[i];
1167
1168                 /* parameter values */
1169                 seq_printf(m, "%s %s%s", field->type, field->name,
1170                            i == event->n_fields - 1 ? "" : "; ");
1171         }
1172
1173         seq_putc(m, '\n');
1174
1175         return 0;
1176 }
1177
1178 static const struct seq_operations synth_events_seq_op = {
1179         .start  = synth_events_seq_start,
1180         .next   = synth_events_seq_next,
1181         .stop   = synth_events_seq_stop,
1182         .show   = synth_events_seq_show
1183 };
1184
1185 static int synth_events_open(struct inode *inode, struct file *file)
1186 {
1187         int ret;
1188
1189         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1190                 ret = release_all_synth_events();
1191                 if (ret < 0)
1192                         return ret;
1193         }
1194
1195         return seq_open(file, &synth_events_seq_op);
1196 }
1197
1198 static ssize_t synth_events_write(struct file *file,
1199                                   const char __user *buffer,
1200                                   size_t count, loff_t *ppos)
1201 {
1202         return trace_parse_run_command(file, buffer, count, ppos,
1203                                        create_synth_event);
1204 }
1205
1206 static const struct file_operations synth_events_fops = {
1207         .open           = synth_events_open,
1208         .write          = synth_events_write,
1209         .read           = seq_read,
1210         .llseek         = seq_lseek,
1211         .release        = seq_release,
1212 };
1213
1214 static u64 hist_field_timestamp(struct hist_field *hist_field,
1215                                 struct tracing_map_elt *elt,
1216                                 struct ring_buffer_event *rbe,
1217                                 void *event)
1218 {
1219         struct hist_trigger_data *hist_data = hist_field->hist_data;
1220         struct trace_array *tr = hist_data->event_file->tr;
1221
1222         u64 ts = ring_buffer_event_time_stamp(rbe);
1223
1224         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1225                 ts = ns2usecs(ts);
1226
1227         return ts;
1228 }
1229
1230 static u64 hist_field_cpu(struct hist_field *hist_field,
1231                           struct tracing_map_elt *elt,
1232                           struct ring_buffer_event *rbe,
1233                           void *event)
1234 {
1235         int cpu = smp_processor_id();
1236
1237         return cpu;
1238 }
1239
1240 static struct hist_field *
1241 check_field_for_var_ref(struct hist_field *hist_field,
1242                         struct hist_trigger_data *var_data,
1243                         unsigned int var_idx)
1244 {
1245         struct hist_field *found = NULL;
1246
1247         if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) {
1248                 if (hist_field->var.idx == var_idx &&
1249                     hist_field->var.hist_data == var_data) {
1250                         found = hist_field;
1251                 }
1252         }
1253
1254         return found;
1255 }
1256
1257 static struct hist_field *
1258 check_field_for_var_refs(struct hist_trigger_data *hist_data,
1259                          struct hist_field *hist_field,
1260                          struct hist_trigger_data *var_data,
1261                          unsigned int var_idx,
1262                          unsigned int level)
1263 {
1264         struct hist_field *found = NULL;
1265         unsigned int i;
1266
1267         if (level > 3)
1268                 return found;
1269
1270         if (!hist_field)
1271                 return found;
1272
1273         found = check_field_for_var_ref(hist_field, var_data, var_idx);
1274         if (found)
1275                 return found;
1276
1277         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1278                 struct hist_field *operand;
1279
1280                 operand = hist_field->operands[i];
1281                 found = check_field_for_var_refs(hist_data, operand, var_data,
1282                                                  var_idx, level + 1);
1283                 if (found)
1284                         return found;
1285         }
1286
1287         return found;
1288 }
1289
1290 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1291                                        struct hist_trigger_data *var_data,
1292                                        unsigned int var_idx)
1293 {
1294         struct hist_field *hist_field, *found = NULL;
1295         unsigned int i;
1296
1297         for_each_hist_field(i, hist_data) {
1298                 hist_field = hist_data->fields[i];
1299                 found = check_field_for_var_refs(hist_data, hist_field,
1300                                                  var_data, var_idx, 0);
1301                 if (found)
1302                         return found;
1303         }
1304
1305         for (i = 0; i < hist_data->n_synth_var_refs; i++) {
1306                 hist_field = hist_data->synth_var_refs[i];
1307                 found = check_field_for_var_refs(hist_data, hist_field,
1308                                                  var_data, var_idx, 0);
1309                 if (found)
1310                         return found;
1311         }
1312
1313         return found;
1314 }
1315
1316 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1317                                            unsigned int var_idx)
1318 {
1319         struct trace_array *tr = hist_data->event_file->tr;
1320         struct hist_field *found = NULL;
1321         struct hist_var_data *var_data;
1322
1323         list_for_each_entry(var_data, &tr->hist_vars, list) {
1324                 if (var_data->hist_data == hist_data)
1325                         continue;
1326                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1327                 if (found)
1328                         break;
1329         }
1330
1331         return found;
1332 }
1333
1334 static bool check_var_refs(struct hist_trigger_data *hist_data)
1335 {
1336         struct hist_field *field;
1337         bool found = false;
1338         int i;
1339
1340         for_each_hist_field(i, hist_data) {
1341                 field = hist_data->fields[i];
1342                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1343                         if (find_any_var_ref(hist_data, field->var.idx)) {
1344                                 found = true;
1345                                 break;
1346                         }
1347                 }
1348         }
1349
1350         return found;
1351 }
1352
1353 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1354 {
1355         struct trace_array *tr = hist_data->event_file->tr;
1356         struct hist_var_data *var_data, *found = NULL;
1357
1358         list_for_each_entry(var_data, &tr->hist_vars, list) {
1359                 if (var_data->hist_data == hist_data) {
1360                         found = var_data;
1361                         break;
1362                 }
1363         }
1364
1365         return found;
1366 }
1367
1368 static bool field_has_hist_vars(struct hist_field *hist_field,
1369                                 unsigned int level)
1370 {
1371         int i;
1372
1373         if (level > 3)
1374                 return false;
1375
1376         if (!hist_field)
1377                 return false;
1378
1379         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1380             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1381                 return true;
1382
1383         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1384                 struct hist_field *operand;
1385
1386                 operand = hist_field->operands[i];
1387                 if (field_has_hist_vars(operand, level + 1))
1388                         return true;
1389         }
1390
1391         return false;
1392 }
1393
1394 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1395 {
1396         struct hist_field *hist_field;
1397         int i;
1398
1399         for_each_hist_field(i, hist_data) {
1400                 hist_field = hist_data->fields[i];
1401                 if (field_has_hist_vars(hist_field, 0))
1402                         return true;
1403         }
1404
1405         return false;
1406 }
1407
1408 static int save_hist_vars(struct hist_trigger_data *hist_data)
1409 {
1410         struct trace_array *tr = hist_data->event_file->tr;
1411         struct hist_var_data *var_data;
1412
1413         var_data = find_hist_vars(hist_data);
1414         if (var_data)
1415                 return 0;
1416
1417         if (trace_array_get(tr) < 0)
1418                 return -ENODEV;
1419
1420         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1421         if (!var_data) {
1422                 trace_array_put(tr);
1423                 return -ENOMEM;
1424         }
1425
1426         var_data->hist_data = hist_data;
1427         list_add(&var_data->list, &tr->hist_vars);
1428
1429         return 0;
1430 }
1431
1432 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1433 {
1434         struct trace_array *tr = hist_data->event_file->tr;
1435         struct hist_var_data *var_data;
1436
1437         var_data = find_hist_vars(hist_data);
1438         if (!var_data)
1439                 return;
1440
1441         if (WARN_ON(check_var_refs(hist_data)))
1442                 return;
1443
1444         list_del(&var_data->list);
1445
1446         kfree(var_data);
1447
1448         trace_array_put(tr);
1449 }
1450
1451 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1452                                          const char *var_name)
1453 {
1454         struct hist_field *hist_field, *found = NULL;
1455         int i;
1456
1457         for_each_hist_field(i, hist_data) {
1458                 hist_field = hist_data->fields[i];
1459                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1460                     strcmp(hist_field->var.name, var_name) == 0) {
1461                         found = hist_field;
1462                         break;
1463                 }
1464         }
1465
1466         return found;
1467 }
1468
1469 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1470                                    struct trace_event_file *file,
1471                                    const char *var_name)
1472 {
1473         struct hist_trigger_data *test_data;
1474         struct event_trigger_data *test;
1475         struct hist_field *hist_field;
1476
1477         hist_field = find_var_field(hist_data, var_name);
1478         if (hist_field)
1479                 return hist_field;
1480
1481         list_for_each_entry_rcu(test, &file->triggers, list) {
1482                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1483                         test_data = test->private_data;
1484                         hist_field = find_var_field(test_data, var_name);
1485                         if (hist_field)
1486                                 return hist_field;
1487                 }
1488         }
1489
1490         return NULL;
1491 }
1492
1493 static struct trace_event_file *find_var_file(struct trace_array *tr,
1494                                               char *system,
1495                                               char *event_name,
1496                                               char *var_name)
1497 {
1498         struct hist_trigger_data *var_hist_data;
1499         struct hist_var_data *var_data;
1500         struct trace_event_file *file, *found = NULL;
1501
1502         if (system)
1503                 return find_event_file(tr, system, event_name);
1504
1505         list_for_each_entry(var_data, &tr->hist_vars, list) {
1506                 var_hist_data = var_data->hist_data;
1507                 file = var_hist_data->event_file;
1508                 if (file == found)
1509                         continue;
1510
1511                 if (find_var_field(var_hist_data, var_name)) {
1512                         if (found) {
1513                                 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1514                                 return NULL;
1515                         }
1516
1517                         found = file;
1518                 }
1519         }
1520
1521         return found;
1522 }
1523
1524 static struct hist_field *find_file_var(struct trace_event_file *file,
1525                                         const char *var_name)
1526 {
1527         struct hist_trigger_data *test_data;
1528         struct event_trigger_data *test;
1529         struct hist_field *hist_field;
1530
1531         list_for_each_entry_rcu(test, &file->triggers, list) {
1532                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1533                         test_data = test->private_data;
1534                         hist_field = find_var_field(test_data, var_name);
1535                         if (hist_field)
1536                                 return hist_field;
1537                 }
1538         }
1539
1540         return NULL;
1541 }
1542
1543 static struct hist_field *
1544 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1545 {
1546         struct trace_array *tr = hist_data->event_file->tr;
1547         struct hist_field *hist_field, *found = NULL;
1548         struct trace_event_file *file;
1549         unsigned int i;
1550
1551         for (i = 0; i < hist_data->n_actions; i++) {
1552                 struct action_data *data = hist_data->actions[i];
1553
1554                 if (data->fn == action_trace) {
1555                         char *system = data->onmatch.match_event_system;
1556                         char *event_name = data->onmatch.match_event;
1557
1558                         file = find_var_file(tr, system, event_name, var_name);
1559                         if (!file)
1560                                 continue;
1561                         hist_field = find_file_var(file, var_name);
1562                         if (hist_field) {
1563                                 if (found) {
1564                                         hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1565                                         return ERR_PTR(-EINVAL);
1566                                 }
1567
1568                                 found = hist_field;
1569                         }
1570                 }
1571         }
1572         return found;
1573 }
1574
1575 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1576                                          char *system,
1577                                          char *event_name,
1578                                          char *var_name)
1579 {
1580         struct trace_array *tr = hist_data->event_file->tr;
1581         struct hist_field *hist_field = NULL;
1582         struct trace_event_file *file;
1583
1584         if (!system || !event_name) {
1585                 hist_field = find_match_var(hist_data, var_name);
1586                 if (IS_ERR(hist_field))
1587                         return NULL;
1588                 if (hist_field)
1589                         return hist_field;
1590         }
1591
1592         file = find_var_file(tr, system, event_name, var_name);
1593         if (!file)
1594                 return NULL;
1595
1596         hist_field = find_file_var(file, var_name);
1597
1598         return hist_field;
1599 }
1600
1601 struct hist_elt_data {
1602         char *comm;
1603         u64 *var_ref_vals;
1604         char *field_var_str[SYNTH_FIELDS_MAX];
1605 };
1606
1607 static u64 hist_field_var_ref(struct hist_field *hist_field,
1608                               struct tracing_map_elt *elt,
1609                               struct ring_buffer_event *rbe,
1610                               void *event)
1611 {
1612         struct hist_elt_data *elt_data;
1613         u64 var_val = 0;
1614
1615         elt_data = elt->private_data;
1616         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1617
1618         return var_val;
1619 }
1620
1621 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1622                              u64 *var_ref_vals, bool self)
1623 {
1624         struct hist_trigger_data *var_data;
1625         struct tracing_map_elt *var_elt;
1626         struct hist_field *hist_field;
1627         unsigned int i, var_idx;
1628         bool resolved = true;
1629         u64 var_val = 0;
1630
1631         for (i = 0; i < hist_data->n_var_refs; i++) {
1632                 hist_field = hist_data->var_refs[i];
1633                 var_idx = hist_field->var.idx;
1634                 var_data = hist_field->var.hist_data;
1635
1636                 if (var_data == NULL) {
1637                         resolved = false;
1638                         break;
1639                 }
1640
1641                 if ((self && var_data != hist_data) ||
1642                     (!self && var_data == hist_data))
1643                         continue;
1644
1645                 var_elt = tracing_map_lookup(var_data->map, key);
1646                 if (!var_elt) {
1647                         resolved = false;
1648                         break;
1649                 }
1650
1651                 if (!tracing_map_var_set(var_elt, var_idx)) {
1652                         resolved = false;
1653                         break;
1654                 }
1655
1656                 if (self || !hist_field->read_once)
1657                         var_val = tracing_map_read_var(var_elt, var_idx);
1658                 else
1659                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1660
1661                 var_ref_vals[i] = var_val;
1662         }
1663
1664         return resolved;
1665 }
1666
1667 static const char *hist_field_name(struct hist_field *field,
1668                                    unsigned int level)
1669 {
1670         const char *field_name = "";
1671
1672         if (level > 1)
1673                 return field_name;
1674
1675         if (field->field)
1676                 field_name = field->field->name;
1677         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1678                  field->flags & HIST_FIELD_FL_ALIAS)
1679                 field_name = hist_field_name(field->operands[0], ++level);
1680         else if (field->flags & HIST_FIELD_FL_CPU)
1681                 field_name = "cpu";
1682         else if (field->flags & HIST_FIELD_FL_EXPR ||
1683                  field->flags & HIST_FIELD_FL_VAR_REF) {
1684                 if (field->system) {
1685                         static char full_name[MAX_FILTER_STR_VAL];
1686
1687                         strcat(full_name, field->system);
1688                         strcat(full_name, ".");
1689                         strcat(full_name, field->event_name);
1690                         strcat(full_name, ".");
1691                         strcat(full_name, field->name);
1692                         field_name = full_name;
1693                 } else
1694                         field_name = field->name;
1695         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1696                 field_name = "common_timestamp";
1697
1698         if (field_name == NULL)
1699                 field_name = "";
1700
1701         return field_name;
1702 }
1703
1704 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1705 {
1706         hist_field_fn_t fn = NULL;
1707
1708         switch (field_size) {
1709         case 8:
1710                 if (field_is_signed)
1711                         fn = hist_field_s64;
1712                 else
1713                         fn = hist_field_u64;
1714                 break;
1715         case 4:
1716                 if (field_is_signed)
1717                         fn = hist_field_s32;
1718                 else
1719                         fn = hist_field_u32;
1720                 break;
1721         case 2:
1722                 if (field_is_signed)
1723                         fn = hist_field_s16;
1724                 else
1725                         fn = hist_field_u16;
1726                 break;
1727         case 1:
1728                 if (field_is_signed)
1729                         fn = hist_field_s8;
1730                 else
1731                         fn = hist_field_u8;
1732                 break;
1733         }
1734
1735         return fn;
1736 }
1737
1738 static int parse_map_size(char *str)
1739 {
1740         unsigned long size, map_bits;
1741         int ret;
1742
1743         strsep(&str, "=");
1744         if (!str) {
1745                 ret = -EINVAL;
1746                 goto out;
1747         }
1748
1749         ret = kstrtoul(str, 0, &size);
1750         if (ret)
1751                 goto out;
1752
1753         map_bits = ilog2(roundup_pow_of_two(size));
1754         if (map_bits < TRACING_MAP_BITS_MIN ||
1755             map_bits > TRACING_MAP_BITS_MAX)
1756                 ret = -EINVAL;
1757         else
1758                 ret = map_bits;
1759  out:
1760         return ret;
1761 }
1762
1763 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1764 {
1765         unsigned int i;
1766
1767         if (!attrs)
1768                 return;
1769
1770         for (i = 0; i < attrs->n_assignments; i++)
1771                 kfree(attrs->assignment_str[i]);
1772
1773         for (i = 0; i < attrs->n_actions; i++)
1774                 kfree(attrs->action_str[i]);
1775
1776         kfree(attrs->name);
1777         kfree(attrs->sort_key_str);
1778         kfree(attrs->keys_str);
1779         kfree(attrs->vals_str);
1780         kfree(attrs->clock);
1781         kfree(attrs);
1782 }
1783
1784 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1785 {
1786         int ret = -EINVAL;
1787
1788         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1789                 return ret;
1790
1791         if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) ||
1792             (strncmp(str, "onmax(", strlen("onmax(")) == 0)) {
1793                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1794                 if (!attrs->action_str[attrs->n_actions]) {
1795                         ret = -ENOMEM;
1796                         return ret;
1797                 }
1798                 attrs->n_actions++;
1799                 ret = 0;
1800         }
1801
1802         return ret;
1803 }
1804
1805 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
1806 {
1807         int ret = 0;
1808
1809         if ((strncmp(str, "key=", strlen("key=")) == 0) ||
1810             (strncmp(str, "keys=", strlen("keys=")) == 0)) {
1811                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
1812                 if (!attrs->keys_str) {
1813                         ret = -ENOMEM;
1814                         goto out;
1815                 }
1816         } else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
1817                  (strncmp(str, "vals=", strlen("vals=")) == 0) ||
1818                  (strncmp(str, "values=", strlen("values=")) == 0)) {
1819                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
1820                 if (!attrs->vals_str) {
1821                         ret = -ENOMEM;
1822                         goto out;
1823                 }
1824         } else if (strncmp(str, "sort=", strlen("sort=")) == 0) {
1825                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
1826                 if (!attrs->sort_key_str) {
1827                         ret = -ENOMEM;
1828                         goto out;
1829                 }
1830         } else if (strncmp(str, "name=", strlen("name=")) == 0) {
1831                 attrs->name = kstrdup(str, GFP_KERNEL);
1832                 if (!attrs->name) {
1833                         ret = -ENOMEM;
1834                         goto out;
1835                 }
1836         } else if (strncmp(str, "clock=", strlen("clock=")) == 0) {
1837                 strsep(&str, "=");
1838                 if (!str) {
1839                         ret = -EINVAL;
1840                         goto out;
1841                 }
1842
1843                 str = strstrip(str);
1844                 attrs->clock = kstrdup(str, GFP_KERNEL);
1845                 if (!attrs->clock) {
1846                         ret = -ENOMEM;
1847                         goto out;
1848                 }
1849         } else if (strncmp(str, "size=", strlen("size=")) == 0) {
1850                 int map_bits = parse_map_size(str);
1851
1852                 if (map_bits < 0) {
1853                         ret = map_bits;
1854                         goto out;
1855                 }
1856                 attrs->map_bits = map_bits;
1857         } else {
1858                 char *assignment;
1859
1860                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1861                         hist_err("Too many variables defined: ", str);
1862                         ret = -EINVAL;
1863                         goto out;
1864                 }
1865
1866                 assignment = kstrdup(str, GFP_KERNEL);
1867                 if (!assignment) {
1868                         ret = -ENOMEM;
1869                         goto out;
1870                 }
1871
1872                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1873         }
1874  out:
1875         return ret;
1876 }
1877
1878 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
1879 {
1880         struct hist_trigger_attrs *attrs;
1881         int ret = 0;
1882
1883         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1884         if (!attrs)
1885                 return ERR_PTR(-ENOMEM);
1886
1887         while (trigger_str) {
1888                 char *str = strsep(&trigger_str, ":");
1889
1890                 if (strchr(str, '=')) {
1891                         ret = parse_assignment(str, attrs);
1892                         if (ret)
1893                                 goto free;
1894                 } else if (strcmp(str, "pause") == 0)
1895                         attrs->pause = true;
1896                 else if ((strcmp(str, "cont") == 0) ||
1897                          (strcmp(str, "continue") == 0))
1898                         attrs->cont = true;
1899                 else if (strcmp(str, "clear") == 0)
1900                         attrs->clear = true;
1901                 else {
1902                         ret = parse_action(str, attrs);
1903                         if (ret)
1904                                 goto free;
1905                 }
1906         }
1907
1908         if (!attrs->keys_str) {
1909                 ret = -EINVAL;
1910                 goto free;
1911         }
1912
1913         if (!attrs->clock) {
1914                 attrs->clock = kstrdup("global", GFP_KERNEL);
1915                 if (!attrs->clock) {
1916                         ret = -ENOMEM;
1917                         goto free;
1918                 }
1919         }
1920
1921         return attrs;
1922  free:
1923         destroy_hist_trigger_attrs(attrs);
1924
1925         return ERR_PTR(ret);
1926 }
1927
1928 static inline void save_comm(char *comm, struct task_struct *task)
1929 {
1930         if (!task->pid) {
1931                 strcpy(comm, "<idle>");
1932                 return;
1933         }
1934
1935         if (WARN_ON_ONCE(task->pid < 0)) {
1936                 strcpy(comm, "<XXX>");
1937                 return;
1938         }
1939
1940         memcpy(comm, task->comm, TASK_COMM_LEN);
1941 }
1942
1943 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1944 {
1945         unsigned int i;
1946
1947         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1948                 kfree(elt_data->field_var_str[i]);
1949
1950         kfree(elt_data->comm);
1951         kfree(elt_data);
1952 }
1953
1954 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1955 {
1956         struct hist_elt_data *elt_data = elt->private_data;
1957
1958         hist_elt_data_free(elt_data);
1959 }
1960
1961 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1962 {
1963         struct hist_trigger_data *hist_data = elt->map->private_data;
1964         unsigned int size = TASK_COMM_LEN;
1965         struct hist_elt_data *elt_data;
1966         struct hist_field *key_field;
1967         unsigned int i, n_str;
1968
1969         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1970         if (!elt_data)
1971                 return -ENOMEM;
1972
1973         for_each_hist_key_field(i, hist_data) {
1974                 key_field = hist_data->fields[i];
1975
1976                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1977                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1978                         if (!elt_data->comm) {
1979                                 kfree(elt_data);
1980                                 return -ENOMEM;
1981                         }
1982                         break;
1983                 }
1984         }
1985
1986         n_str = hist_data->n_field_var_str + hist_data->n_max_var_str;
1987
1988         size = STR_VAR_LEN_MAX;
1989
1990         for (i = 0; i < n_str; i++) {
1991                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1992                 if (!elt_data->field_var_str[i]) {
1993                         hist_elt_data_free(elt_data);
1994                         return -ENOMEM;
1995                 }
1996         }
1997
1998         elt->private_data = elt_data;
1999
2000         return 0;
2001 }
2002
2003 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2004 {
2005         struct hist_elt_data *elt_data = elt->private_data;
2006
2007         if (elt_data->comm)
2008                 save_comm(elt_data->comm, current);
2009 }
2010
2011 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2012         .elt_alloc      = hist_trigger_elt_data_alloc,
2013         .elt_free       = hist_trigger_elt_data_free,
2014         .elt_init       = hist_trigger_elt_data_init,
2015 };
2016
2017 static const char *get_hist_field_flags(struct hist_field *hist_field)
2018 {
2019         const char *flags_str = NULL;
2020
2021         if (hist_field->flags & HIST_FIELD_FL_HEX)
2022                 flags_str = "hex";
2023         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2024                 flags_str = "sym";
2025         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2026                 flags_str = "sym-offset";
2027         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2028                 flags_str = "execname";
2029         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2030                 flags_str = "syscall";
2031         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2032                 flags_str = "log2";
2033         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2034                 flags_str = "usecs";
2035
2036         return flags_str;
2037 }
2038
2039 static void expr_field_str(struct hist_field *field, char *expr)
2040 {
2041         if (field->flags & HIST_FIELD_FL_VAR_REF)
2042                 strcat(expr, "$");
2043
2044         strcat(expr, hist_field_name(field, 0));
2045
2046         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2047                 const char *flags_str = get_hist_field_flags(field);
2048
2049                 if (flags_str) {
2050                         strcat(expr, ".");
2051                         strcat(expr, flags_str);
2052                 }
2053         }
2054 }
2055
2056 static char *expr_str(struct hist_field *field, unsigned int level)
2057 {
2058         char *expr;
2059
2060         if (level > 1)
2061                 return NULL;
2062
2063         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2064         if (!expr)
2065                 return NULL;
2066
2067         if (!field->operands[0]) {
2068                 expr_field_str(field, expr);
2069                 return expr;
2070         }
2071
2072         if (field->operator == FIELD_OP_UNARY_MINUS) {
2073                 char *subexpr;
2074
2075                 strcat(expr, "-(");
2076                 subexpr = expr_str(field->operands[0], ++level);
2077                 if (!subexpr) {
2078                         kfree(expr);
2079                         return NULL;
2080                 }
2081                 strcat(expr, subexpr);
2082                 strcat(expr, ")");
2083
2084                 kfree(subexpr);
2085
2086                 return expr;
2087         }
2088
2089         expr_field_str(field->operands[0], expr);
2090
2091         switch (field->operator) {
2092         case FIELD_OP_MINUS:
2093                 strcat(expr, "-");
2094                 break;
2095         case FIELD_OP_PLUS:
2096                 strcat(expr, "+");
2097                 break;
2098         default:
2099                 kfree(expr);
2100                 return NULL;
2101         }
2102
2103         expr_field_str(field->operands[1], expr);
2104
2105         return expr;
2106 }
2107
2108 static int contains_operator(char *str)
2109 {
2110         enum field_op_id field_op = FIELD_OP_NONE;
2111         char *op;
2112
2113         op = strpbrk(str, "+-");
2114         if (!op)
2115                 return FIELD_OP_NONE;
2116
2117         switch (*op) {
2118         case '-':
2119                 if (*str == '-')
2120                         field_op = FIELD_OP_UNARY_MINUS;
2121                 else
2122                         field_op = FIELD_OP_MINUS;
2123                 break;
2124         case '+':
2125                 field_op = FIELD_OP_PLUS;
2126                 break;
2127         default:
2128                 break;
2129         }
2130
2131         return field_op;
2132 }
2133
2134 static void destroy_hist_field(struct hist_field *hist_field,
2135                                unsigned int level)
2136 {
2137         unsigned int i;
2138
2139         if (level > 3)
2140                 return;
2141
2142         if (!hist_field)
2143                 return;
2144
2145         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2146                 destroy_hist_field(hist_field->operands[i], level + 1);
2147
2148         kfree(hist_field->var.name);
2149         kfree(hist_field->name);
2150         kfree(hist_field->type);
2151
2152         kfree(hist_field);
2153 }
2154
2155 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2156                                             struct ftrace_event_field *field,
2157                                             unsigned long flags,
2158                                             char *var_name)
2159 {
2160         struct hist_field *hist_field;
2161
2162         if (field && is_function_field(field))
2163                 return NULL;
2164
2165         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2166         if (!hist_field)
2167                 return NULL;
2168
2169         hist_field->hist_data = hist_data;
2170
2171         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2172                 goto out; /* caller will populate */
2173
2174         if (flags & HIST_FIELD_FL_VAR_REF) {
2175                 hist_field->fn = hist_field_var_ref;
2176                 goto out;
2177         }
2178
2179         if (flags & HIST_FIELD_FL_HITCOUNT) {
2180                 hist_field->fn = hist_field_counter;
2181                 hist_field->size = sizeof(u64);
2182                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2183                 if (!hist_field->type)
2184                         goto free;
2185                 goto out;
2186         }
2187
2188         if (flags & HIST_FIELD_FL_STACKTRACE) {
2189                 hist_field->fn = hist_field_none;
2190                 goto out;
2191         }
2192
2193         if (flags & HIST_FIELD_FL_LOG2) {
2194                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2195                 hist_field->fn = hist_field_log2;
2196                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2197                 hist_field->size = hist_field->operands[0]->size;
2198                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2199                 if (!hist_field->type)
2200                         goto free;
2201                 goto out;
2202         }
2203
2204         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2205                 hist_field->fn = hist_field_timestamp;
2206                 hist_field->size = sizeof(u64);
2207                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2208                 if (!hist_field->type)
2209                         goto free;
2210                 goto out;
2211         }
2212
2213         if (flags & HIST_FIELD_FL_CPU) {
2214                 hist_field->fn = hist_field_cpu;
2215                 hist_field->size = sizeof(int);
2216                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2217                 if (!hist_field->type)
2218                         goto free;
2219                 goto out;
2220         }
2221
2222         if (WARN_ON_ONCE(!field))
2223                 goto out;
2224
2225         if (is_string_field(field)) {
2226                 flags |= HIST_FIELD_FL_STRING;
2227
2228                 hist_field->size = MAX_FILTER_STR_VAL;
2229                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2230                 if (!hist_field->type)
2231                         goto free;
2232
2233                 if (field->filter_type == FILTER_STATIC_STRING)
2234                         hist_field->fn = hist_field_string;
2235                 else if (field->filter_type == FILTER_DYN_STRING)
2236                         hist_field->fn = hist_field_dynstring;
2237                 else
2238                         hist_field->fn = hist_field_pstring;
2239         } else {
2240                 hist_field->size = field->size;
2241                 hist_field->is_signed = field->is_signed;
2242                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2243                 if (!hist_field->type)
2244                         goto free;
2245
2246                 hist_field->fn = select_value_fn(field->size,
2247                                                  field->is_signed);
2248                 if (!hist_field->fn) {
2249                         destroy_hist_field(hist_field, 0);
2250                         return NULL;
2251                 }
2252         }
2253  out:
2254         hist_field->field = field;
2255         hist_field->flags = flags;
2256
2257         if (var_name) {
2258                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2259                 if (!hist_field->var.name)
2260                         goto free;
2261         }
2262
2263         return hist_field;
2264  free:
2265         destroy_hist_field(hist_field, 0);
2266         return NULL;
2267 }
2268
2269 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2270 {
2271         unsigned int i;
2272
2273         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2274                 if (hist_data->fields[i]) {
2275                         destroy_hist_field(hist_data->fields[i], 0);
2276                         hist_data->fields[i] = NULL;
2277                 }
2278         }
2279 }
2280
2281 static int init_var_ref(struct hist_field *ref_field,
2282                         struct hist_field *var_field,
2283                         char *system, char *event_name)
2284 {
2285         int err = 0;
2286
2287         ref_field->var.idx = var_field->var.idx;
2288         ref_field->var.hist_data = var_field->hist_data;
2289         ref_field->size = var_field->size;
2290         ref_field->is_signed = var_field->is_signed;
2291         ref_field->flags |= var_field->flags &
2292                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2293
2294         if (system) {
2295                 ref_field->system = kstrdup(system, GFP_KERNEL);
2296                 if (!ref_field->system)
2297                         return -ENOMEM;
2298         }
2299
2300         if (event_name) {
2301                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2302                 if (!ref_field->event_name) {
2303                         err = -ENOMEM;
2304                         goto free;
2305                 }
2306         }
2307
2308         if (var_field->var.name) {
2309                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2310                 if (!ref_field->name) {
2311                         err = -ENOMEM;
2312                         goto free;
2313                 }
2314         } else if (var_field->name) {
2315                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2316                 if (!ref_field->name) {
2317                         err = -ENOMEM;
2318                         goto free;
2319                 }
2320         }
2321
2322         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2323         if (!ref_field->type) {
2324                 err = -ENOMEM;
2325                 goto free;
2326         }
2327  out:
2328         return err;
2329  free:
2330         kfree(ref_field->system);
2331         kfree(ref_field->event_name);
2332         kfree(ref_field->name);
2333
2334         goto out;
2335 }
2336
2337 static struct hist_field *create_var_ref(struct hist_field *var_field,
2338                                          char *system, char *event_name)
2339 {
2340         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2341         struct hist_field *ref_field;
2342
2343         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2344         if (ref_field) {
2345                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2346                         destroy_hist_field(ref_field, 0);
2347                         return NULL;
2348                 }
2349         }
2350
2351         return ref_field;
2352 }
2353
2354 static bool is_var_ref(char *var_name)
2355 {
2356         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2357                 return false;
2358
2359         return true;
2360 }
2361
2362 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2363                                  char *var_name)
2364 {
2365         char *name, *field;
2366         unsigned int i;
2367
2368         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2369                 name = hist_data->attrs->var_defs.name[i];
2370
2371                 if (strcmp(var_name, name) == 0) {
2372                         field = hist_data->attrs->var_defs.expr[i];
2373                         if (contains_operator(field) || is_var_ref(field))
2374                                 continue;
2375                         return field;
2376                 }
2377         }
2378
2379         return NULL;
2380 }
2381
2382 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2383                                  char *system, char *event_name,
2384                                  char *var_name)
2385 {
2386         struct trace_event_call *call;
2387
2388         if (system && event_name) {
2389                 call = hist_data->event_file->event_call;
2390
2391                 if (strcmp(system, call->class->system) != 0)
2392                         return NULL;
2393
2394                 if (strcmp(event_name, trace_event_name(call)) != 0)
2395                         return NULL;
2396         }
2397
2398         if (!!system != !!event_name)
2399                 return NULL;
2400
2401         if (!is_var_ref(var_name))
2402                 return NULL;
2403
2404         var_name++;
2405
2406         return field_name_from_var(hist_data, var_name);
2407 }
2408
2409 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2410                                         char *system, char *event_name,
2411                                         char *var_name)
2412 {
2413         struct hist_field *var_field = NULL, *ref_field = NULL;
2414
2415         if (!is_var_ref(var_name))
2416                 return NULL;
2417
2418         var_name++;
2419
2420         var_field = find_event_var(hist_data, system, event_name, var_name);
2421         if (var_field)
2422                 ref_field = create_var_ref(var_field, system, event_name);
2423
2424         if (!ref_field)
2425                 hist_err_event("Couldn't find variable: $",
2426                                system, event_name, var_name);
2427
2428         return ref_field;
2429 }
2430
2431 static struct ftrace_event_field *
2432 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2433             char *field_str, unsigned long *flags)
2434 {
2435         struct ftrace_event_field *field = NULL;
2436         char *field_name, *modifier, *str;
2437
2438         modifier = str = kstrdup(field_str, GFP_KERNEL);
2439         if (!modifier)
2440                 return ERR_PTR(-ENOMEM);
2441
2442         field_name = strsep(&modifier, ".");
2443         if (modifier) {
2444                 if (strcmp(modifier, "hex") == 0)
2445                         *flags |= HIST_FIELD_FL_HEX;
2446                 else if (strcmp(modifier, "sym") == 0)
2447                         *flags |= HIST_FIELD_FL_SYM;
2448                 else if (strcmp(modifier, "sym-offset") == 0)
2449                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2450                 else if ((strcmp(modifier, "execname") == 0) &&
2451                          (strcmp(field_name, "common_pid") == 0))
2452                         *flags |= HIST_FIELD_FL_EXECNAME;
2453                 else if (strcmp(modifier, "syscall") == 0)
2454                         *flags |= HIST_FIELD_FL_SYSCALL;
2455                 else if (strcmp(modifier, "log2") == 0)
2456                         *flags |= HIST_FIELD_FL_LOG2;
2457                 else if (strcmp(modifier, "usecs") == 0)
2458                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2459                 else {
2460                         hist_err("Invalid field modifier: ", modifier);
2461                         field = ERR_PTR(-EINVAL);
2462                         goto out;
2463                 }
2464         }
2465
2466         if (strcmp(field_name, "common_timestamp") == 0) {
2467                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2468                 hist_data->enable_timestamps = true;
2469                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2470                         hist_data->attrs->ts_in_usecs = true;
2471         } else if (strcmp(field_name, "cpu") == 0)
2472                 *flags |= HIST_FIELD_FL_CPU;
2473         else {
2474                 field = trace_find_event_field(file->event_call, field_name);
2475                 if (!field || !field->size) {
2476                         hist_err("Couldn't find field: ", field_name);
2477                         field = ERR_PTR(-EINVAL);
2478                         goto out;
2479                 }
2480         }
2481  out:
2482         kfree(str);
2483
2484         return field;
2485 }
2486
2487 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2488                                        struct hist_field *var_ref,
2489                                        char *var_name)
2490 {
2491         struct hist_field *alias = NULL;
2492         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2493
2494         alias = create_hist_field(hist_data, NULL, flags, var_name);
2495         if (!alias)
2496                 return NULL;
2497
2498         alias->fn = var_ref->fn;
2499         alias->operands[0] = var_ref;
2500
2501         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2502                 destroy_hist_field(alias, 0);
2503                 return NULL;
2504         }
2505
2506         return alias;
2507 }
2508
2509 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2510                                      struct trace_event_file *file, char *str,
2511                                      unsigned long *flags, char *var_name)
2512 {
2513         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2514         struct ftrace_event_field *field = NULL;
2515         struct hist_field *hist_field = NULL;
2516         int ret = 0;
2517
2518         s = strchr(str, '.');
2519         if (s) {
2520                 s = strchr(++s, '.');
2521                 if (s) {
2522                         ref_system = strsep(&str, ".");
2523                         if (!str) {
2524                                 ret = -EINVAL;
2525                                 goto out;
2526                         }
2527                         ref_event = strsep(&str, ".");
2528                         if (!str) {
2529                                 ret = -EINVAL;
2530                                 goto out;
2531                         }
2532                         ref_var = str;
2533                 }
2534         }
2535
2536         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2537         if (!s) {
2538                 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2539                 if (hist_field) {
2540                         hist_data->var_refs[hist_data->n_var_refs] = hist_field;
2541                         hist_field->var_ref_idx = hist_data->n_var_refs++;
2542                         if (var_name) {
2543                                 hist_field = create_alias(hist_data, hist_field, var_name);
2544                                 if (!hist_field) {
2545                                         ret = -ENOMEM;
2546                                         goto out;
2547                                 }
2548                         }
2549                         return hist_field;
2550                 }
2551         } else
2552                 str = s;
2553
2554         field = parse_field(hist_data, file, str, flags);
2555         if (IS_ERR(field)) {
2556                 ret = PTR_ERR(field);
2557                 goto out;
2558         }
2559
2560         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2561         if (!hist_field) {
2562                 ret = -ENOMEM;
2563                 goto out;
2564         }
2565
2566         return hist_field;
2567  out:
2568         return ERR_PTR(ret);
2569 }
2570
2571 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2572                                      struct trace_event_file *file,
2573                                      char *str, unsigned long flags,
2574                                      char *var_name, unsigned int level);
2575
2576 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2577                                       struct trace_event_file *file,
2578                                       char *str, unsigned long flags,
2579                                       char *var_name, unsigned int level)
2580 {
2581         struct hist_field *operand1, *expr = NULL;
2582         unsigned long operand_flags;
2583         int ret = 0;
2584         char *s;
2585
2586         /* we support only -(xxx) i.e. explicit parens required */
2587
2588         if (level > 3) {
2589                 hist_err("Too many subexpressions (3 max): ", str);
2590                 ret = -EINVAL;
2591                 goto free;
2592         }
2593
2594         str++; /* skip leading '-' */
2595
2596         s = strchr(str, '(');
2597         if (s)
2598                 str++;
2599         else {
2600                 ret = -EINVAL;
2601                 goto free;
2602         }
2603
2604         s = strrchr(str, ')');
2605         if (s)
2606                 *s = '\0';
2607         else {
2608                 ret = -EINVAL; /* no closing ')' */
2609                 goto free;
2610         }
2611
2612         flags |= HIST_FIELD_FL_EXPR;
2613         expr = create_hist_field(hist_data, NULL, flags, var_name);
2614         if (!expr) {
2615                 ret = -ENOMEM;
2616                 goto free;
2617         }
2618
2619         operand_flags = 0;
2620         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2621         if (IS_ERR(operand1)) {
2622                 ret = PTR_ERR(operand1);
2623                 goto free;
2624         }
2625
2626         expr->flags |= operand1->flags &
2627                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2628         expr->fn = hist_field_unary_minus;
2629         expr->operands[0] = operand1;
2630         expr->operator = FIELD_OP_UNARY_MINUS;
2631         expr->name = expr_str(expr, 0);
2632         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2633         if (!expr->type) {
2634                 ret = -ENOMEM;
2635                 goto free;
2636         }
2637
2638         return expr;
2639  free:
2640         destroy_hist_field(expr, 0);
2641         return ERR_PTR(ret);
2642 }
2643
2644 static int check_expr_operands(struct hist_field *operand1,
2645                                struct hist_field *operand2)
2646 {
2647         unsigned long operand1_flags = operand1->flags;
2648         unsigned long operand2_flags = operand2->flags;
2649
2650         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2651             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2652                 struct hist_field *var;
2653
2654                 var = find_var_field(operand1->var.hist_data, operand1->name);
2655                 if (!var)
2656                         return -EINVAL;
2657                 operand1_flags = var->flags;
2658         }
2659
2660         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2661             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2662                 struct hist_field *var;
2663
2664                 var = find_var_field(operand2->var.hist_data, operand2->name);
2665                 if (!var)
2666                         return -EINVAL;
2667                 operand2_flags = var->flags;
2668         }
2669
2670         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2671             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2672                 hist_err("Timestamp units in expression don't match", NULL);
2673                 return -EINVAL;
2674         }
2675
2676         return 0;
2677 }
2678
2679 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2680                                      struct trace_event_file *file,
2681                                      char *str, unsigned long flags,
2682                                      char *var_name, unsigned int level)
2683 {
2684         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2685         unsigned long operand_flags;
2686         int field_op, ret = -EINVAL;
2687         char *sep, *operand1_str;
2688
2689         if (level > 3) {
2690                 hist_err("Too many subexpressions (3 max): ", str);
2691                 return ERR_PTR(-EINVAL);
2692         }
2693
2694         field_op = contains_operator(str);
2695
2696         if (field_op == FIELD_OP_NONE)
2697                 return parse_atom(hist_data, file, str, &flags, var_name);
2698
2699         if (field_op == FIELD_OP_UNARY_MINUS)
2700                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2701
2702         switch (field_op) {
2703         case FIELD_OP_MINUS:
2704                 sep = "-";
2705                 break;
2706         case FIELD_OP_PLUS:
2707                 sep = "+";
2708                 break;
2709         default:
2710                 goto free;
2711         }
2712
2713         operand1_str = strsep(&str, sep);
2714         if (!operand1_str || !str)
2715                 goto free;
2716
2717         operand_flags = 0;
2718         operand1 = parse_atom(hist_data, file, operand1_str,
2719                               &operand_flags, NULL);
2720         if (IS_ERR(operand1)) {
2721                 ret = PTR_ERR(operand1);
2722                 operand1 = NULL;
2723                 goto free;
2724         }
2725
2726         /* rest of string could be another expression e.g. b+c in a+b+c */
2727         operand_flags = 0;
2728         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2729         if (IS_ERR(operand2)) {
2730                 ret = PTR_ERR(operand2);
2731                 operand2 = NULL;
2732                 goto free;
2733         }
2734
2735         ret = check_expr_operands(operand1, operand2);
2736         if (ret)
2737                 goto free;
2738
2739         flags |= HIST_FIELD_FL_EXPR;
2740
2741         flags |= operand1->flags &
2742                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2743
2744         expr = create_hist_field(hist_data, NULL, flags, var_name);
2745         if (!expr) {
2746                 ret = -ENOMEM;
2747                 goto free;
2748         }
2749
2750         operand1->read_once = true;
2751         operand2->read_once = true;
2752
2753         expr->operands[0] = operand1;
2754         expr->operands[1] = operand2;
2755         expr->operator = field_op;
2756         expr->name = expr_str(expr, 0);
2757         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2758         if (!expr->type) {
2759                 ret = -ENOMEM;
2760                 goto free;
2761         }
2762
2763         switch (field_op) {
2764         case FIELD_OP_MINUS:
2765                 expr->fn = hist_field_minus;
2766                 break;
2767         case FIELD_OP_PLUS:
2768                 expr->fn = hist_field_plus;
2769                 break;
2770         default:
2771                 ret = -EINVAL;
2772                 goto free;
2773         }
2774
2775         return expr;
2776  free:
2777         destroy_hist_field(operand1, 0);
2778         destroy_hist_field(operand2, 0);
2779         destroy_hist_field(expr, 0);
2780
2781         return ERR_PTR(ret);
2782 }
2783
2784 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2785                                  struct trace_event_file *file)
2786 {
2787         struct event_trigger_data *test;
2788
2789         list_for_each_entry_rcu(test, &file->triggers, list) {
2790                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2791                         if (test->private_data == hist_data)
2792                                 return test->filter_str;
2793                 }
2794         }
2795
2796         return NULL;
2797 }
2798
2799 static struct event_command trigger_hist_cmd;
2800 static int event_hist_trigger_func(struct event_command *cmd_ops,
2801                                    struct trace_event_file *file,
2802                                    char *glob, char *cmd, char *param);
2803
2804 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2805                             struct hist_trigger_data *hist_data,
2806                             unsigned int n_keys)
2807 {
2808         struct hist_field *target_hist_field, *hist_field;
2809         unsigned int n, i, j;
2810
2811         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2812                 return false;
2813
2814         i = hist_data->n_vals;
2815         j = target_hist_data->n_vals;
2816
2817         for (n = 0; n < n_keys; n++) {
2818                 hist_field = hist_data->fields[i + n];
2819                 target_hist_field = target_hist_data->fields[j + n];
2820
2821                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2822                         return false;
2823                 if (hist_field->size != target_hist_field->size)
2824                         return false;
2825                 if (hist_field->is_signed != target_hist_field->is_signed)
2826                         return false;
2827         }
2828
2829         return true;
2830 }
2831
2832 static struct hist_trigger_data *
2833 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2834                      struct trace_event_file *file)
2835 {
2836         struct hist_trigger_data *hist_data;
2837         struct event_trigger_data *test;
2838         unsigned int n_keys;
2839
2840         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2841
2842         list_for_each_entry_rcu(test, &file->triggers, list) {
2843                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2844                         hist_data = test->private_data;
2845
2846                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2847                                 return hist_data;
2848                 }
2849         }
2850
2851         return NULL;
2852 }
2853
2854 static struct trace_event_file *event_file(struct trace_array *tr,
2855                                            char *system, char *event_name)
2856 {
2857         struct trace_event_file *file;
2858
2859         file = __find_event_file(tr, system, event_name);
2860         if (!file)
2861                 return ERR_PTR(-EINVAL);
2862
2863         return file;
2864 }
2865
2866 static struct hist_field *
2867 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2868                          char *system, char *event_name, char *field_name)
2869 {
2870         struct hist_field *event_var;
2871         char *synthetic_name;
2872
2873         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2874         if (!synthetic_name)
2875                 return ERR_PTR(-ENOMEM);
2876
2877         strcpy(synthetic_name, "synthetic_");
2878         strcat(synthetic_name, field_name);
2879
2880         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2881
2882         kfree(synthetic_name);
2883
2884         return event_var;
2885 }
2886
2887 /**
2888  * create_field_var_hist - Automatically create a histogram and var for a field
2889  * @target_hist_data: The target hist trigger
2890  * @subsys_name: Optional subsystem name
2891  * @event_name: Optional event name
2892  * @field_name: The name of the field (and the resulting variable)
2893  *
2894  * Hist trigger actions fetch data from variables, not directly from
2895  * events.  However, for convenience, users are allowed to directly
2896  * specify an event field in an action, which will be automatically
2897  * converted into a variable on their behalf.
2898
2899  * If a user specifies a field on an event that isn't the event the
2900  * histogram currently being defined (the target event histogram), the
2901  * only way that can be accomplished is if a new hist trigger is
2902  * created and the field variable defined on that.
2903  *
2904  * This function creates a new histogram compatible with the target
2905  * event (meaning a histogram with the same key as the target
2906  * histogram), and creates a variable for the specified field, but
2907  * with 'synthetic_' prepended to the variable name in order to avoid
2908  * collision with normal field variables.
2909  *
2910  * Return: The variable created for the field.
2911  */
2912 static struct hist_field *
2913 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2914                       char *subsys_name, char *event_name, char *field_name)
2915 {
2916         struct trace_array *tr = target_hist_data->event_file->tr;
2917         struct hist_field *event_var = ERR_PTR(-EINVAL);
2918         struct hist_trigger_data *hist_data;
2919         unsigned int i, n, first = true;
2920         struct field_var_hist *var_hist;
2921         struct trace_event_file *file;
2922         struct hist_field *key_field;
2923         char *saved_filter;
2924         char *cmd;
2925         int ret;
2926
2927         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2928                 hist_err_event("onmatch: Too many field variables defined: ",
2929                                subsys_name, event_name, field_name);
2930                 return ERR_PTR(-EINVAL);
2931         }
2932
2933         file = event_file(tr, subsys_name, event_name);
2934
2935         if (IS_ERR(file)) {
2936                 hist_err_event("onmatch: Event file not found: ",
2937                                subsys_name, event_name, field_name);
2938                 ret = PTR_ERR(file);
2939                 return ERR_PTR(ret);
2940         }
2941
2942         /*
2943          * Look for a histogram compatible with target.  We'll use the
2944          * found histogram specification to create a new matching
2945          * histogram with our variable on it.  target_hist_data is not
2946          * yet a registered histogram so we can't use that.
2947          */
2948         hist_data = find_compatible_hist(target_hist_data, file);
2949         if (!hist_data) {
2950                 hist_err_event("onmatch: Matching event histogram not found: ",
2951                                subsys_name, event_name, field_name);
2952                 return ERR_PTR(-EINVAL);
2953         }
2954
2955         /* See if a synthetic field variable has already been created */
2956         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2957                                              event_name, field_name);
2958         if (!IS_ERR_OR_NULL(event_var))
2959                 return event_var;
2960
2961         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2962         if (!var_hist)
2963                 return ERR_PTR(-ENOMEM);
2964
2965         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2966         if (!cmd) {
2967                 kfree(var_hist);
2968                 return ERR_PTR(-ENOMEM);
2969         }
2970
2971         /* Use the same keys as the compatible histogram */
2972         strcat(cmd, "keys=");
2973
2974         for_each_hist_key_field(i, hist_data) {
2975                 key_field = hist_data->fields[i];
2976                 if (!first)
2977                         strcat(cmd, ",");
2978                 strcat(cmd, key_field->field->name);
2979                 first = false;
2980         }
2981
2982         /* Create the synthetic field variable specification */
2983         strcat(cmd, ":synthetic_");
2984         strcat(cmd, field_name);
2985         strcat(cmd, "=");
2986         strcat(cmd, field_name);
2987
2988         /* Use the same filter as the compatible histogram */
2989         saved_filter = find_trigger_filter(hist_data, file);
2990         if (saved_filter) {
2991                 strcat(cmd, " if ");
2992                 strcat(cmd, saved_filter);
2993         }
2994
2995         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2996         if (!var_hist->cmd) {
2997                 kfree(cmd);
2998                 kfree(var_hist);
2999                 return ERR_PTR(-ENOMEM);
3000         }
3001
3002         /* Save the compatible histogram information */
3003         var_hist->hist_data = hist_data;
3004
3005         /* Create the new histogram with our variable */
3006         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3007                                       "", "hist", cmd);
3008         if (ret) {
3009                 kfree(cmd);
3010                 kfree(var_hist->cmd);
3011                 kfree(var_hist);
3012                 hist_err_event("onmatch: Couldn't create histogram for field: ",
3013                                subsys_name, event_name, field_name);
3014                 return ERR_PTR(ret);
3015         }
3016
3017         kfree(cmd);
3018
3019         /* If we can't find the variable, something went wrong */
3020         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3021                                              event_name, field_name);
3022         if (IS_ERR_OR_NULL(event_var)) {
3023                 kfree(var_hist->cmd);
3024                 kfree(var_hist);
3025                 hist_err_event("onmatch: Couldn't find synthetic variable: ",
3026                                subsys_name, event_name, field_name);
3027                 return ERR_PTR(-EINVAL);
3028         }
3029
3030         n = target_hist_data->n_field_var_hists;
3031         target_hist_data->field_var_hists[n] = var_hist;
3032         target_hist_data->n_field_var_hists++;
3033
3034         return event_var;
3035 }
3036
3037 static struct hist_field *
3038 find_target_event_var(struct hist_trigger_data *hist_data,
3039                       char *subsys_name, char *event_name, char *var_name)
3040 {
3041         struct trace_event_file *file = hist_data->event_file;
3042         struct hist_field *hist_field = NULL;
3043
3044         if (subsys_name) {
3045                 struct trace_event_call *call;
3046
3047                 if (!event_name)
3048                         return NULL;
3049
3050                 call = file->event_call;
3051
3052                 if (strcmp(subsys_name, call->class->system) != 0)
3053                         return NULL;
3054
3055                 if (strcmp(event_name, trace_event_name(call)) != 0)
3056                         return NULL;
3057         }
3058
3059         hist_field = find_var_field(hist_data, var_name);
3060
3061         return hist_field;
3062 }
3063
3064 static inline void __update_field_vars(struct tracing_map_elt *elt,
3065                                        struct ring_buffer_event *rbe,
3066                                        void *rec,
3067                                        struct field_var **field_vars,
3068                                        unsigned int n_field_vars,
3069                                        unsigned int field_var_str_start)
3070 {
3071         struct hist_elt_data *elt_data = elt->private_data;
3072         unsigned int i, j, var_idx;
3073         u64 var_val;
3074
3075         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3076                 struct field_var *field_var = field_vars[i];
3077                 struct hist_field *var = field_var->var;
3078                 struct hist_field *val = field_var->val;
3079
3080                 var_val = val->fn(val, elt, rbe, rec);
3081                 var_idx = var->var.idx;
3082
3083                 if (val->flags & HIST_FIELD_FL_STRING) {
3084                         char *str = elt_data->field_var_str[j++];
3085                         char *val_str = (char *)(uintptr_t)var_val;
3086
3087                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3088                         var_val = (u64)(uintptr_t)str;
3089                 }
3090                 tracing_map_set_var(elt, var_idx, var_val);
3091         }
3092 }
3093
3094 static void update_field_vars(struct hist_trigger_data *hist_data,
3095                               struct tracing_map_elt *elt,
3096                               struct ring_buffer_event *rbe,
3097                               void *rec)
3098 {
3099         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3100                             hist_data->n_field_vars, 0);
3101 }
3102
3103 static void update_max_vars(struct hist_trigger_data *hist_data,
3104                             struct tracing_map_elt *elt,
3105                             struct ring_buffer_event *rbe,
3106                             void *rec)
3107 {
3108         __update_field_vars(elt, rbe, rec, hist_data->max_vars,
3109                             hist_data->n_max_vars, hist_data->n_field_var_str);
3110 }
3111
3112 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3113                                      struct trace_event_file *file,
3114                                      char *name, int size, const char *type)
3115 {
3116         struct hist_field *var;
3117         int idx;
3118
3119         if (find_var(hist_data, file, name) && !hist_data->remove) {
3120                 var = ERR_PTR(-EINVAL);
3121                 goto out;
3122         }
3123
3124         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3125         if (!var) {
3126                 var = ERR_PTR(-ENOMEM);
3127                 goto out;
3128         }
3129
3130         idx = tracing_map_add_var(hist_data->map);
3131         if (idx < 0) {
3132                 kfree(var);
3133                 var = ERR_PTR(-EINVAL);
3134                 goto out;
3135         }
3136
3137         var->flags = HIST_FIELD_FL_VAR;
3138         var->var.idx = idx;
3139         var->var.hist_data = var->hist_data = hist_data;
3140         var->size = size;
3141         var->var.name = kstrdup(name, GFP_KERNEL);
3142         var->type = kstrdup(type, GFP_KERNEL);
3143         if (!var->var.name || !var->type) {
3144                 kfree(var->var.name);
3145                 kfree(var->type);
3146                 kfree(var);
3147                 var = ERR_PTR(-ENOMEM);
3148         }
3149  out:
3150         return var;
3151 }
3152
3153 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3154                                           struct trace_event_file *file,
3155                                           char *field_name)
3156 {
3157         struct hist_field *val = NULL, *var = NULL;
3158         unsigned long flags = HIST_FIELD_FL_VAR;
3159         struct field_var *field_var;
3160         int ret = 0;
3161
3162         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3163                 hist_err("Too many field variables defined: ", field_name);
3164                 ret = -EINVAL;
3165                 goto err;
3166         }
3167
3168         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3169         if (IS_ERR(val)) {
3170                 hist_err("Couldn't parse field variable: ", field_name);
3171                 ret = PTR_ERR(val);
3172                 goto err;
3173         }
3174
3175         var = create_var(hist_data, file, field_name, val->size, val->type);
3176         if (IS_ERR(var)) {
3177                 hist_err("Couldn't create or find variable: ", field_name);
3178                 kfree(val);
3179                 ret = PTR_ERR(var);
3180                 goto err;
3181         }
3182
3183         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3184         if (!field_var) {
3185                 kfree(val);
3186                 kfree(var);
3187                 ret =  -ENOMEM;
3188                 goto err;
3189         }
3190
3191         field_var->var = var;
3192         field_var->val = val;
3193  out:
3194         return field_var;
3195  err:
3196         field_var = ERR_PTR(ret);
3197         goto out;
3198 }
3199
3200 /**
3201  * create_target_field_var - Automatically create a variable for a field
3202  * @target_hist_data: The target hist trigger
3203  * @subsys_name: Optional subsystem name
3204  * @event_name: Optional event name
3205  * @var_name: The name of the field (and the resulting variable)
3206  *
3207  * Hist trigger actions fetch data from variables, not directly from
3208  * events.  However, for convenience, users are allowed to directly
3209  * specify an event field in an action, which will be automatically
3210  * converted into a variable on their behalf.
3211
3212  * This function creates a field variable with the name var_name on
3213  * the hist trigger currently being defined on the target event.  If
3214  * subsys_name and event_name are specified, this function simply
3215  * verifies that they do in fact match the target event subsystem and
3216  * event name.
3217  *
3218  * Return: The variable created for the field.
3219  */
3220 static struct field_var *
3221 create_target_field_var(struct hist_trigger_data *target_hist_data,
3222                         char *subsys_name, char *event_name, char *var_name)
3223 {
3224         struct trace_event_file *file = target_hist_data->event_file;
3225
3226         if (subsys_name) {
3227                 struct trace_event_call *call;
3228
3229                 if (!event_name)
3230                         return NULL;
3231
3232                 call = file->event_call;
3233
3234                 if (strcmp(subsys_name, call->class->system) != 0)
3235                         return NULL;
3236
3237                 if (strcmp(event_name, trace_event_name(call)) != 0)
3238                         return NULL;
3239         }
3240
3241         return create_field_var(target_hist_data, file, var_name);
3242 }
3243
3244 static void onmax_print(struct seq_file *m,
3245                         struct hist_trigger_data *hist_data,
3246                         struct tracing_map_elt *elt,
3247                         struct action_data *data)
3248 {
3249         unsigned int i, save_var_idx, max_idx = data->onmax.max_var->var.idx;
3250
3251         seq_printf(m, "\n\tmax: %10llu", tracing_map_read_var(elt, max_idx));
3252
3253         for (i = 0; i < hist_data->n_max_vars; i++) {
3254                 struct hist_field *save_val = hist_data->max_vars[i]->val;
3255                 struct hist_field *save_var = hist_data->max_vars[i]->var;
3256                 u64 val;
3257
3258                 save_var_idx = save_var->var.idx;
3259
3260                 val = tracing_map_read_var(elt, save_var_idx);
3261
3262                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3263                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3264                                    (char *)(uintptr_t)(val));
3265                 } else
3266                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3267         }
3268 }
3269
3270 static void onmax_save(struct hist_trigger_data *hist_data,
3271                        struct tracing_map_elt *elt, void *rec,
3272                        struct ring_buffer_event *rbe,
3273                        struct action_data *data, u64 *var_ref_vals)
3274 {
3275         unsigned int max_idx = data->onmax.max_var->var.idx;
3276         unsigned int max_var_ref_idx = data->onmax.max_var_ref_idx;
3277
3278         u64 var_val, max_val;
3279
3280         var_val = var_ref_vals[max_var_ref_idx];
3281         max_val = tracing_map_read_var(elt, max_idx);
3282
3283         if (var_val <= max_val)
3284                 return;
3285
3286         tracing_map_set_var(elt, max_idx, var_val);
3287
3288         update_max_vars(hist_data, elt, rbe, rec);
3289 }
3290
3291 static void onmax_destroy(struct action_data *data)
3292 {
3293         unsigned int i;
3294
3295         destroy_hist_field(data->onmax.max_var, 0);
3296         destroy_hist_field(data->onmax.var, 0);
3297
3298         kfree(data->onmax.var_str);
3299         kfree(data->onmax.fn_name);
3300
3301         for (i = 0; i < data->n_params; i++)
3302                 kfree(data->params[i]);
3303
3304         kfree(data);
3305 }
3306
3307 static int onmax_create(struct hist_trigger_data *hist_data,
3308                         struct action_data *data)
3309 {
3310         struct trace_event_file *file = hist_data->event_file;
3311         struct hist_field *var_field, *ref_field, *max_var;
3312         unsigned int var_ref_idx = hist_data->n_var_refs;
3313         struct field_var *field_var;
3314         char *onmax_var_str, *param;
3315         unsigned long flags;
3316         unsigned int i;
3317         int ret = 0;
3318
3319         onmax_var_str = data->onmax.var_str;
3320         if (onmax_var_str[0] != '$') {
3321                 hist_err("onmax: For onmax(x), x must be a variable: ", onmax_var_str);
3322                 return -EINVAL;
3323         }
3324         onmax_var_str++;
3325
3326         var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str);
3327         if (!var_field) {
3328                 hist_err("onmax: Couldn't find onmax variable: ", onmax_var_str);
3329                 return -EINVAL;
3330         }
3331
3332         flags = HIST_FIELD_FL_VAR_REF;
3333         ref_field = create_hist_field(hist_data, NULL, flags, NULL);
3334         if (!ref_field)
3335                 return -ENOMEM;
3336
3337         if (init_var_ref(ref_field, var_field, NULL, NULL)) {
3338                 destroy_hist_field(ref_field, 0);
3339                 ret = -ENOMEM;
3340                 goto out;
3341         }
3342         hist_data->var_refs[hist_data->n_var_refs] = ref_field;
3343         ref_field->var_ref_idx = hist_data->n_var_refs++;
3344         data->onmax.var = ref_field;
3345
3346         data->fn = onmax_save;
3347         data->onmax.max_var_ref_idx = var_ref_idx;
3348         max_var = create_var(hist_data, file, "max", sizeof(u64), "u64");
3349         if (IS_ERR(max_var)) {
3350                 hist_err("onmax: Couldn't create onmax variable: ", "max");
3351                 ret = PTR_ERR(max_var);
3352                 goto out;
3353         }
3354         data->onmax.max_var = max_var;
3355
3356         for (i = 0; i < data->n_params; i++) {
3357                 param = kstrdup(data->params[i], GFP_KERNEL);
3358                 if (!param) {
3359                         ret = -ENOMEM;
3360                         goto out;
3361                 }
3362
3363                 field_var = create_target_field_var(hist_data, NULL, NULL, param);
3364                 if (IS_ERR(field_var)) {
3365                         hist_err("onmax: Couldn't create field variable: ", param);
3366                         ret = PTR_ERR(field_var);
3367                         kfree(param);
3368                         goto out;
3369                 }
3370
3371                 hist_data->max_vars[hist_data->n_max_vars++] = field_var;
3372                 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3373                         hist_data->n_max_var_str++;
3374
3375                 kfree(param);
3376         }
3377  out:
3378         return ret;
3379 }
3380
3381 static int parse_action_params(char *params, struct action_data *data)
3382 {
3383         char *param, *saved_param;
3384         int ret = 0;
3385
3386         while (params) {
3387                 if (data->n_params >= SYNTH_FIELDS_MAX)
3388                         goto out;
3389
3390                 param = strsep(&params, ",");
3391                 if (!param) {
3392                         ret = -EINVAL;
3393                         goto out;
3394                 }
3395
3396                 param = strstrip(param);
3397                 if (strlen(param) < 2) {
3398                         hist_err("Invalid action param: ", param);
3399                         ret = -EINVAL;
3400                         goto out;
3401                 }
3402
3403                 saved_param = kstrdup(param, GFP_KERNEL);
3404                 if (!saved_param) {
3405                         ret = -ENOMEM;
3406                         goto out;
3407                 }
3408
3409                 data->params[data->n_params++] = saved_param;
3410         }
3411  out:
3412         return ret;
3413 }
3414
3415 static struct action_data *onmax_parse(char *str)
3416 {
3417         char *onmax_fn_name, *onmax_var_str;
3418         struct action_data *data;
3419         int ret = -EINVAL;
3420
3421         data = kzalloc(sizeof(*data), GFP_KERNEL);
3422         if (!data)
3423                 return ERR_PTR(-ENOMEM);
3424
3425         onmax_var_str = strsep(&str, ")");
3426         if (!onmax_var_str || !str) {
3427                 ret = -EINVAL;
3428                 goto free;
3429         }
3430
3431         data->onmax.var_str = kstrdup(onmax_var_str, GFP_KERNEL);
3432         if (!data->onmax.var_str) {
3433                 ret = -ENOMEM;
3434                 goto free;
3435         }
3436
3437         strsep(&str, ".");
3438         if (!str)
3439                 goto free;
3440
3441         onmax_fn_name = strsep(&str, "(");
3442         if (!onmax_fn_name || !str)
3443                 goto free;
3444
3445         if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) {
3446                 char *params = strsep(&str, ")");
3447
3448                 if (!params) {
3449                         ret = -EINVAL;
3450                         goto free;
3451                 }
3452
3453                 ret = parse_action_params(params, data);
3454                 if (ret)
3455                         goto free;
3456         } else
3457                 goto free;
3458
3459         data->onmax.fn_name = kstrdup(onmax_fn_name, GFP_KERNEL);
3460         if (!data->onmax.fn_name) {
3461                 ret = -ENOMEM;
3462                 goto free;
3463         }
3464  out:
3465         return data;
3466  free:
3467         onmax_destroy(data);
3468         data = ERR_PTR(ret);
3469         goto out;
3470 }
3471
3472 static void onmatch_destroy(struct action_data *data)
3473 {
3474         unsigned int i;
3475
3476         mutex_lock(&synth_event_mutex);
3477
3478         kfree(data->onmatch.match_event);
3479         kfree(data->onmatch.match_event_system);
3480         kfree(data->onmatch.synth_event_name);
3481
3482         for (i = 0; i < data->n_params; i++)
3483                 kfree(data->params[i]);
3484
3485         if (data->onmatch.synth_event)
3486                 data->onmatch.synth_event->ref--;
3487
3488         kfree(data);
3489
3490         mutex_unlock(&synth_event_mutex);
3491 }
3492
3493 static void destroy_field_var(struct field_var *field_var)
3494 {
3495         if (!field_var)
3496                 return;
3497
3498         destroy_hist_field(field_var->var, 0);
3499         destroy_hist_field(field_var->val, 0);
3500
3501         kfree(field_var);
3502 }
3503
3504 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3505 {
3506         unsigned int i;
3507
3508         for (i = 0; i < hist_data->n_field_vars; i++)
3509                 destroy_field_var(hist_data->field_vars[i]);
3510 }
3511
3512 static void save_field_var(struct hist_trigger_data *hist_data,
3513                            struct field_var *field_var)
3514 {
3515         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3516
3517         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3518                 hist_data->n_field_var_str++;
3519 }
3520
3521
3522 static void destroy_synth_var_refs(struct hist_trigger_data *hist_data)
3523 {
3524         unsigned int i;
3525
3526         for (i = 0; i < hist_data->n_synth_var_refs; i++)
3527                 destroy_hist_field(hist_data->synth_var_refs[i], 0);
3528 }
3529
3530 static void save_synth_var_ref(struct hist_trigger_data *hist_data,
3531                          struct hist_field *var_ref)
3532 {
3533         hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref;
3534
3535         hist_data->var_refs[hist_data->n_var_refs] = var_ref;
3536         var_ref->var_ref_idx = hist_data->n_var_refs++;
3537 }
3538
3539 static int check_synth_field(struct synth_event *event,
3540                              struct hist_field *hist_field,
3541                              unsigned int field_pos)
3542 {
3543         struct synth_field *field;
3544
3545         if (field_pos >= event->n_fields)
3546                 return -EINVAL;
3547
3548         field = event->fields[field_pos];
3549
3550         if (strcmp(field->type, hist_field->type) != 0)
3551                 return -EINVAL;
3552
3553         return 0;
3554 }
3555
3556 static struct hist_field *
3557 onmatch_find_var(struct hist_trigger_data *hist_data, struct action_data *data,
3558                  char *system, char *event, char *var)
3559 {
3560         struct hist_field *hist_field;
3561
3562         var++; /* skip '$' */
3563
3564         hist_field = find_target_event_var(hist_data, system, event, var);
3565         if (!hist_field) {
3566                 if (!system) {
3567                         system = data->onmatch.match_event_system;
3568                         event = data->onmatch.match_event;
3569                 }
3570
3571                 hist_field = find_event_var(hist_data, system, event, var);
3572         }
3573
3574         if (!hist_field)
3575                 hist_err_event("onmatch: Couldn't find onmatch param: $", system, event, var);
3576
3577         return hist_field;
3578 }
3579
3580 static struct hist_field *
3581 onmatch_create_field_var(struct hist_trigger_data *hist_data,
3582                          struct action_data *data, char *system,
3583                          char *event, char *var)
3584 {
3585         struct hist_field *hist_field = NULL;
3586         struct field_var *field_var;
3587
3588         /*
3589          * First try to create a field var on the target event (the
3590          * currently being defined).  This will create a variable for
3591          * unqualified fields on the target event, or if qualified,
3592          * target fields that have qualified names matching the target.
3593          */
3594         field_var = create_target_field_var(hist_data, system, event, var);
3595
3596         if (field_var && !IS_ERR(field_var)) {
3597                 save_field_var(hist_data, field_var);
3598                 hist_field = field_var->var;
3599         } else {
3600                 field_var = NULL;
3601                 /*
3602                  * If no explicit system.event is specfied, default to
3603                  * looking for fields on the onmatch(system.event.xxx)
3604                  * event.
3605                  */
3606                 if (!system) {
3607                         system = data->onmatch.match_event_system;
3608                         event = data->onmatch.match_event;
3609                 }
3610
3611                 /*
3612                  * At this point, we're looking at a field on another
3613                  * event.  Because we can't modify a hist trigger on
3614                  * another event to add a variable for a field, we need
3615                  * to create a new trigger on that event and create the
3616                  * variable at the same time.
3617                  */
3618                 hist_field = create_field_var_hist(hist_data, system, event, var);
3619                 if (IS_ERR(hist_field))
3620                         goto free;
3621         }
3622  out:
3623         return hist_field;
3624  free:
3625         destroy_field_var(field_var);
3626         hist_field = NULL;
3627         goto out;
3628 }
3629
3630 static int onmatch_create(struct hist_trigger_data *hist_data,
3631                           struct trace_event_file *file,
3632                           struct action_data *data)
3633 {
3634         char *event_name, *param, *system = NULL;
3635         struct hist_field *hist_field, *var_ref;
3636         unsigned int i, var_ref_idx;
3637         unsigned int field_pos = 0;
3638         struct synth_event *event;
3639         int ret = 0;
3640
3641         mutex_lock(&synth_event_mutex);
3642         event = find_synth_event(data->onmatch.synth_event_name);
3643         if (!event) {
3644                 hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name);
3645                 mutex_unlock(&synth_event_mutex);
3646                 return -EINVAL;
3647         }
3648         event->ref++;
3649         mutex_unlock(&synth_event_mutex);
3650
3651         var_ref_idx = hist_data->n_var_refs;
3652
3653         for (i = 0; i < data->n_params; i++) {
3654                 char *p;
3655
3656                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3657                 if (!param) {
3658                         ret = -ENOMEM;
3659                         goto err;
3660                 }
3661
3662                 system = strsep(&param, ".");
3663                 if (!param) {
3664                         param = (char *)system;
3665                         system = event_name = NULL;
3666                 } else {
3667                         event_name = strsep(&param, ".");
3668                         if (!param) {
3669                                 kfree(p);
3670                                 ret = -EINVAL;
3671                                 goto err;
3672                         }
3673                 }
3674
3675                 if (param[0] == '$')
3676                         hist_field = onmatch_find_var(hist_data, data, system,
3677                                                       event_name, param);
3678                 else
3679                         hist_field = onmatch_create_field_var(hist_data, data,
3680                                                               system,
3681                                                               event_name,
3682                                                               param);
3683
3684                 if (!hist_field) {
3685                         kfree(p);
3686                         ret = -EINVAL;
3687                         goto err;
3688                 }
3689
3690                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3691                         var_ref = create_var_ref(hist_field, system, event_name);
3692                         if (!var_ref) {
3693                                 kfree(p);
3694                                 ret = -ENOMEM;
3695                                 goto err;
3696                         }
3697
3698                         save_synth_var_ref(hist_data, var_ref);
3699                         field_pos++;
3700                         kfree(p);
3701                         continue;
3702                 }
3703
3704                 hist_err_event("onmatch: Param type doesn't match synthetic event field type: ",
3705                                system, event_name, param);
3706                 kfree(p);
3707                 ret = -EINVAL;
3708                 goto err;
3709         }
3710
3711         if (field_pos != event->n_fields) {
3712                 hist_err("onmatch: Param count doesn't match synthetic event field count: ", event->name);
3713                 ret = -EINVAL;
3714                 goto err;
3715         }
3716
3717         data->fn = action_trace;
3718         data->onmatch.synth_event = event;
3719         data->onmatch.var_ref_idx = var_ref_idx;
3720  out:
3721         return ret;
3722  err:
3723         mutex_lock(&synth_event_mutex);
3724         event->ref--;
3725         mutex_unlock(&synth_event_mutex);
3726
3727         goto out;
3728 }
3729
3730 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3731 {
3732         char *match_event, *match_event_system;
3733         char *synth_event_name, *params;
3734         struct action_data *data;
3735         int ret = -EINVAL;
3736
3737         data = kzalloc(sizeof(*data), GFP_KERNEL);
3738         if (!data)
3739                 return ERR_PTR(-ENOMEM);
3740
3741         match_event = strsep(&str, ")");
3742         if (!match_event || !str) {
3743                 hist_err("onmatch: Missing closing paren: ", match_event);
3744                 goto free;
3745         }
3746
3747         match_event_system = strsep(&match_event, ".");
3748         if (!match_event) {
3749                 hist_err("onmatch: Missing subsystem for match event: ", match_event_system);
3750                 goto free;
3751         }
3752
3753         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3754                 hist_err_event("onmatch: Invalid subsystem or event name: ",
3755                                match_event_system, match_event, NULL);
3756                 goto free;
3757         }
3758
3759         data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL);
3760         if (!data->onmatch.match_event) {
3761                 ret = -ENOMEM;
3762                 goto free;
3763         }
3764
3765         data->onmatch.match_event_system = kstrdup(match_event_system, GFP_KERNEL);
3766         if (!data->onmatch.match_event_system) {
3767                 ret = -ENOMEM;
3768                 goto free;
3769         }
3770
3771         strsep(&str, ".");
3772         if (!str) {
3773                 hist_err("onmatch: Missing . after onmatch(): ", str);
3774                 goto free;
3775         }
3776
3777         synth_event_name = strsep(&str, "(");
3778         if (!synth_event_name || !str) {
3779                 hist_err("onmatch: Missing opening paramlist paren: ", synth_event_name);
3780                 goto free;
3781         }
3782
3783         data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL);
3784         if (!data->onmatch.synth_event_name) {
3785                 ret = -ENOMEM;
3786                 goto free;
3787         }
3788
3789         params = strsep(&str, ")");
3790         if (!params || !str || (str && strlen(str))) {
3791                 hist_err("onmatch: Missing closing paramlist paren: ", params);
3792                 goto free;
3793         }
3794
3795         ret = parse_action_params(params, data);
3796         if (ret)
3797                 goto free;
3798  out:
3799         return data;
3800  free:
3801         onmatch_destroy(data);
3802         data = ERR_PTR(ret);
3803         goto out;
3804 }
3805
3806 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3807 {
3808         hist_data->fields[HITCOUNT_IDX] =
3809                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3810         if (!hist_data->fields[HITCOUNT_IDX])
3811                 return -ENOMEM;
3812
3813         hist_data->n_vals++;
3814         hist_data->n_fields++;
3815
3816         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3817                 return -EINVAL;
3818
3819         return 0;
3820 }
3821
3822 static int __create_val_field(struct hist_trigger_data *hist_data,
3823                               unsigned int val_idx,
3824                               struct trace_event_file *file,
3825                               char *var_name, char *field_str,
3826                               unsigned long flags)
3827 {
3828         struct hist_field *hist_field;
3829         int ret = 0;
3830
3831         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3832         if (IS_ERR(hist_field)) {
3833                 ret = PTR_ERR(hist_field);
3834                 goto out;
3835         }
3836
3837         hist_data->fields[val_idx] = hist_field;
3838
3839         ++hist_data->n_vals;
3840         ++hist_data->n_fields;
3841
3842         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3843                 ret = -EINVAL;
3844  out:
3845         return ret;
3846 }
3847
3848 static int create_val_field(struct hist_trigger_data *hist_data,
3849                             unsigned int val_idx,
3850                             struct trace_event_file *file,
3851                             char *field_str)
3852 {
3853         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3854                 return -EINVAL;
3855
3856         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3857 }
3858
3859 static int create_var_field(struct hist_trigger_data *hist_data,
3860                             unsigned int val_idx,
3861                             struct trace_event_file *file,
3862                             char *var_name, char *expr_str)
3863 {
3864         unsigned long flags = 0;
3865
3866         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3867                 return -EINVAL;
3868
3869         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3870                 hist_err("Variable already defined: ", var_name);
3871                 return -EINVAL;
3872         }
3873
3874         flags |= HIST_FIELD_FL_VAR;
3875         hist_data->n_vars++;
3876         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3877                 return -EINVAL;
3878
3879         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3880 }
3881
3882 static int create_val_fields(struct hist_trigger_data *hist_data,
3883                              struct trace_event_file *file)
3884 {
3885         char *fields_str, *field_str;
3886         unsigned int i, j = 1;
3887         int ret;
3888
3889         ret = create_hitcount_val(hist_data);
3890         if (ret)
3891                 goto out;
3892
3893         fields_str = hist_data->attrs->vals_str;
3894         if (!fields_str)
3895                 goto out;
3896
3897         strsep(&fields_str, "=");
3898         if (!fields_str)
3899                 goto out;
3900
3901         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3902                      j < TRACING_MAP_VALS_MAX; i++) {
3903                 field_str = strsep(&fields_str, ",");
3904                 if (!field_str)
3905                         break;
3906
3907                 if (strcmp(field_str, "hitcount") == 0)
3908                         continue;
3909
3910                 ret = create_val_field(hist_data, j++, file, field_str);
3911                 if (ret)
3912                         goto out;
3913         }
3914
3915         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3916                 ret = -EINVAL;
3917  out:
3918         return ret;
3919 }
3920
3921 static int create_key_field(struct hist_trigger_data *hist_data,
3922                             unsigned int key_idx,
3923                             unsigned int key_offset,
3924                             struct trace_event_file *file,
3925                             char *field_str)
3926 {
3927         struct hist_field *hist_field = NULL;
3928
3929         unsigned long flags = 0;
3930         unsigned int key_size;
3931         int ret = 0;
3932
3933         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3934                 return -EINVAL;
3935
3936         flags |= HIST_FIELD_FL_KEY;
3937
3938         if (strcmp(field_str, "stacktrace") == 0) {
3939                 flags |= HIST_FIELD_FL_STACKTRACE;
3940                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3941                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3942         } else {
3943                 hist_field = parse_expr(hist_data, file, field_str, flags,
3944                                         NULL, 0);
3945                 if (IS_ERR(hist_field)) {
3946                         ret = PTR_ERR(hist_field);
3947                         goto out;
3948                 }
3949
3950                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
3951                         hist_err("Using variable references as keys not supported: ", field_str);
3952                         destroy_hist_field(hist_field, 0);
3953                         ret = -EINVAL;
3954                         goto out;
3955                 }
3956
3957                 key_size = hist_field->size;
3958         }
3959
3960         hist_data->fields[key_idx] = hist_field;
3961
3962         key_size = ALIGN(key_size, sizeof(u64));
3963         hist_data->fields[key_idx]->size = key_size;
3964         hist_data->fields[key_idx]->offset = key_offset;
3965
3966         hist_data->key_size += key_size;
3967
3968         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3969                 ret = -EINVAL;
3970                 goto out;
3971         }
3972
3973         hist_data->n_keys++;
3974         hist_data->n_fields++;
3975
3976         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3977                 return -EINVAL;
3978
3979         ret = key_size;
3980  out:
3981         return ret;
3982 }
3983
3984 static int create_key_fields(struct hist_trigger_data *hist_data,
3985                              struct trace_event_file *file)
3986 {
3987         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3988         char *fields_str, *field_str;
3989         int ret = -EINVAL;
3990
3991         fields_str = hist_data->attrs->keys_str;
3992         if (!fields_str)
3993                 goto out;
3994
3995         strsep(&fields_str, "=");
3996         if (!fields_str)
3997                 goto out;
3998
3999         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4000                 field_str = strsep(&fields_str, ",");
4001                 if (!field_str)
4002                         break;
4003                 ret = create_key_field(hist_data, i, key_offset,
4004                                        file, field_str);
4005                 if (ret < 0)
4006                         goto out;
4007                 key_offset += ret;
4008         }
4009         if (fields_str) {
4010                 ret = -EINVAL;
4011                 goto out;
4012         }
4013         ret = 0;
4014  out:
4015         return ret;
4016 }
4017
4018 static int create_var_fields(struct hist_trigger_data *hist_data,
4019                              struct trace_event_file *file)
4020 {
4021         unsigned int i, j = hist_data->n_vals;
4022         int ret = 0;
4023
4024         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4025
4026         for (i = 0; i < n_vars; i++) {
4027                 char *var_name = hist_data->attrs->var_defs.name[i];
4028                 char *expr = hist_data->attrs->var_defs.expr[i];
4029
4030                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4031                 if (ret)
4032                         goto out;
4033         }
4034  out:
4035         return ret;
4036 }
4037
4038 static void free_var_defs(struct hist_trigger_data *hist_data)
4039 {
4040         unsigned int i;
4041
4042         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4043                 kfree(hist_data->attrs->var_defs.name[i]);
4044                 kfree(hist_data->attrs->var_defs.expr[i]);
4045         }
4046
4047         hist_data->attrs->var_defs.n_vars = 0;
4048 }
4049
4050 static int parse_var_defs(struct hist_trigger_data *hist_data)
4051 {
4052         char *s, *str, *var_name, *field_str;
4053         unsigned int i, j, n_vars = 0;
4054         int ret = 0;
4055
4056         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4057                 str = hist_data->attrs->assignment_str[i];
4058                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4059                         field_str = strsep(&str, ",");
4060                         if (!field_str)
4061                                 break;
4062
4063                         var_name = strsep(&field_str, "=");
4064                         if (!var_name || !field_str) {
4065                                 hist_err("Malformed assignment: ", var_name);
4066                                 ret = -EINVAL;
4067                                 goto free;
4068                         }
4069
4070                         if (n_vars == TRACING_MAP_VARS_MAX) {
4071                                 hist_err("Too many variables defined: ", var_name);
4072                                 ret = -EINVAL;
4073                                 goto free;
4074                         }
4075
4076                         s = kstrdup(var_name, GFP_KERNEL);
4077                         if (!s) {
4078                                 ret = -ENOMEM;
4079                                 goto free;
4080                         }
4081                         hist_data->attrs->var_defs.name[n_vars] = s;
4082
4083                         s = kstrdup(field_str, GFP_KERNEL);
4084                         if (!s) {
4085                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4086                                 ret = -ENOMEM;
4087                                 goto free;
4088                         }
4089                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4090
4091                         hist_data->attrs->var_defs.n_vars = n_vars;
4092                 }
4093         }
4094
4095         return ret;
4096  free:
4097         free_var_defs(hist_data);
4098
4099         return ret;
4100 }
4101
4102 static int create_hist_fields(struct hist_trigger_data *hist_data,
4103                               struct trace_event_file *file)
4104 {
4105         int ret;
4106
4107         ret = parse_var_defs(hist_data);
4108         if (ret)
4109                 goto out;
4110
4111         ret = create_val_fields(hist_data, file);
4112         if (ret)
4113                 goto out;
4114
4115         ret = create_var_fields(hist_data, file);
4116         if (ret)
4117                 goto out;
4118
4119         ret = create_key_fields(hist_data, file);
4120         if (ret)
4121                 goto out;
4122  out:
4123         free_var_defs(hist_data);
4124
4125         return ret;
4126 }
4127
4128 static int is_descending(const char *str)
4129 {
4130         if (!str)
4131                 return 0;
4132
4133         if (strcmp(str, "descending") == 0)
4134                 return 1;
4135
4136         if (strcmp(str, "ascending") == 0)
4137                 return 0;
4138
4139         return -EINVAL;
4140 }
4141
4142 static int create_sort_keys(struct hist_trigger_data *hist_data)
4143 {
4144         char *fields_str = hist_data->attrs->sort_key_str;
4145         struct tracing_map_sort_key *sort_key;
4146         int descending, ret = 0;
4147         unsigned int i, j, k;
4148
4149         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4150
4151         if (!fields_str)
4152                 goto out;
4153
4154         strsep(&fields_str, "=");
4155         if (!fields_str) {
4156                 ret = -EINVAL;
4157                 goto out;
4158         }
4159
4160         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4161                 struct hist_field *hist_field;
4162                 char *field_str, *field_name;
4163                 const char *test_name;
4164
4165                 sort_key = &hist_data->sort_keys[i];
4166
4167                 field_str = strsep(&fields_str, ",");
4168                 if (!field_str) {
4169                         if (i == 0)
4170                                 ret = -EINVAL;
4171                         break;
4172                 }
4173
4174                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4175                         ret = -EINVAL;
4176                         break;
4177                 }
4178
4179                 field_name = strsep(&field_str, ".");
4180                 if (!field_name) {
4181                         ret = -EINVAL;
4182                         break;
4183                 }
4184
4185                 if (strcmp(field_name, "hitcount") == 0) {
4186                         descending = is_descending(field_str);
4187                         if (descending < 0) {
4188                                 ret = descending;
4189                                 break;
4190                         }
4191                         sort_key->descending = descending;
4192                         continue;
4193                 }
4194
4195                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4196                         unsigned int idx;
4197
4198                         hist_field = hist_data->fields[j];
4199                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4200                                 continue;
4201
4202                         idx = k++;
4203
4204                         test_name = hist_field_name(hist_field, 0);
4205
4206                         if (strcmp(field_name, test_name) == 0) {
4207                                 sort_key->field_idx = idx;
4208                                 descending = is_descending(field_str);
4209                                 if (descending < 0) {
4210                                         ret = descending;
4211                                         goto out;
4212                                 }
4213                                 sort_key->descending = descending;
4214                                 break;
4215                         }
4216                 }
4217                 if (j == hist_data->n_fields) {
4218                         ret = -EINVAL;
4219                         break;
4220                 }
4221         }
4222
4223         hist_data->n_sort_keys = i;
4224  out:
4225         return ret;
4226 }
4227
4228 static void destroy_actions(struct hist_trigger_data *hist_data)
4229 {
4230         unsigned int i;
4231
4232         for (i = 0; i < hist_data->n_actions; i++) {
4233                 struct action_data *data = hist_data->actions[i];
4234
4235                 if (data->fn == action_trace)
4236                         onmatch_destroy(data);
4237                 else if (data->fn == onmax_save)
4238                         onmax_destroy(data);
4239                 else
4240                         kfree(data);
4241         }
4242 }
4243
4244 static int parse_actions(struct hist_trigger_data *hist_data)
4245 {
4246         struct trace_array *tr = hist_data->event_file->tr;
4247         struct action_data *data;
4248         unsigned int i;
4249         int ret = 0;
4250         char *str;
4251
4252         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4253                 str = hist_data->attrs->action_str[i];
4254
4255                 if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) {
4256                         char *action_str = str + strlen("onmatch(");
4257
4258                         data = onmatch_parse(tr, action_str);
4259                         if (IS_ERR(data)) {
4260                                 ret = PTR_ERR(data);
4261                                 break;
4262                         }
4263                         data->fn = action_trace;
4264                 } else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) {
4265                         char *action_str = str + strlen("onmax(");
4266
4267                         data = onmax_parse(action_str);
4268                         if (IS_ERR(data)) {
4269                                 ret = PTR_ERR(data);
4270                                 break;
4271                         }
4272                         data->fn = onmax_save;
4273                 } else {
4274                         ret = -EINVAL;
4275                         break;
4276                 }
4277
4278                 hist_data->actions[hist_data->n_actions++] = data;
4279         }
4280
4281         return ret;
4282 }
4283
4284 static int create_actions(struct hist_trigger_data *hist_data,
4285                           struct trace_event_file *file)
4286 {
4287         struct action_data *data;
4288         unsigned int i;
4289         int ret = 0;
4290
4291         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4292                 data = hist_data->actions[i];
4293
4294                 if (data->fn == action_trace) {
4295                         ret = onmatch_create(hist_data, file, data);
4296                         if (ret)
4297                                 return ret;
4298                 } else if (data->fn == onmax_save) {
4299                         ret = onmax_create(hist_data, data);
4300                         if (ret)
4301                                 return ret;
4302                 }
4303         }
4304
4305         return ret;
4306 }
4307
4308 static void print_actions(struct seq_file *m,
4309                           struct hist_trigger_data *hist_data,
4310                           struct tracing_map_elt *elt)
4311 {
4312         unsigned int i;
4313
4314         for (i = 0; i < hist_data->n_actions; i++) {
4315                 struct action_data *data = hist_data->actions[i];
4316
4317                 if (data->fn == onmax_save)
4318                         onmax_print(m, hist_data, elt, data);
4319         }
4320 }
4321
4322 static void print_onmax_spec(struct seq_file *m,
4323                              struct hist_trigger_data *hist_data,
4324                              struct action_data *data)
4325 {
4326         unsigned int i;
4327
4328         seq_puts(m, ":onmax(");
4329         seq_printf(m, "%s", data->onmax.var_str);
4330         seq_printf(m, ").%s(", data->onmax.fn_name);
4331
4332         for (i = 0; i < hist_data->n_max_vars; i++) {
4333                 seq_printf(m, "%s", hist_data->max_vars[i]->var->var.name);
4334                 if (i < hist_data->n_max_vars - 1)
4335                         seq_puts(m, ",");
4336         }
4337         seq_puts(m, ")");
4338 }
4339
4340 static void print_onmatch_spec(struct seq_file *m,
4341                                struct hist_trigger_data *hist_data,
4342                                struct action_data *data)
4343 {
4344         unsigned int i;
4345
4346         seq_printf(m, ":onmatch(%s.%s).", data->onmatch.match_event_system,
4347                    data->onmatch.match_event);
4348
4349         seq_printf(m, "%s(", data->onmatch.synth_event->name);
4350
4351         for (i = 0; i < data->n_params; i++) {
4352                 if (i)
4353                         seq_puts(m, ",");
4354                 seq_printf(m, "%s", data->params[i]);
4355         }
4356
4357         seq_puts(m, ")");
4358 }
4359
4360 static bool actions_match(struct hist_trigger_data *hist_data,
4361                           struct hist_trigger_data *hist_data_test)
4362 {
4363         unsigned int i, j;
4364
4365         if (hist_data->n_actions != hist_data_test->n_actions)
4366                 return false;
4367
4368         for (i = 0; i < hist_data->n_actions; i++) {
4369                 struct action_data *data = hist_data->actions[i];
4370                 struct action_data *data_test = hist_data_test->actions[i];
4371
4372                 if (data->fn != data_test->fn)
4373                         return false;
4374
4375                 if (data->n_params != data_test->n_params)
4376                         return false;
4377
4378                 for (j = 0; j < data->n_params; j++) {
4379                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4380                                 return false;
4381                 }
4382
4383                 if (data->fn == action_trace) {
4384                         if (strcmp(data->onmatch.synth_event_name,
4385                                    data_test->onmatch.synth_event_name) != 0)
4386                                 return false;
4387                         if (strcmp(data->onmatch.match_event_system,
4388                                    data_test->onmatch.match_event_system) != 0)
4389                                 return false;
4390                         if (strcmp(data->onmatch.match_event,
4391                                    data_test->onmatch.match_event) != 0)
4392                                 return false;
4393                 } else if (data->fn == onmax_save) {
4394                         if (strcmp(data->onmax.var_str,
4395                                    data_test->onmax.var_str) != 0)
4396                                 return false;
4397                         if (strcmp(data->onmax.fn_name,
4398                                    data_test->onmax.fn_name) != 0)
4399                                 return false;
4400                 }
4401         }
4402
4403         return true;
4404 }
4405
4406
4407 static void print_actions_spec(struct seq_file *m,
4408                                struct hist_trigger_data *hist_data)
4409 {
4410         unsigned int i;
4411
4412         for (i = 0; i < hist_data->n_actions; i++) {
4413                 struct action_data *data = hist_data->actions[i];
4414
4415                 if (data->fn == action_trace)
4416                         print_onmatch_spec(m, hist_data, data);
4417                 else if (data->fn == onmax_save)
4418                         print_onmax_spec(m, hist_data, data);
4419         }
4420 }
4421
4422 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4423 {
4424         unsigned int i;
4425
4426         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4427                 kfree(hist_data->field_var_hists[i]->cmd);
4428                 kfree(hist_data->field_var_hists[i]);
4429         }
4430 }
4431
4432 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4433 {
4434         if (!hist_data)
4435                 return;
4436
4437         destroy_hist_trigger_attrs(hist_data->attrs);
4438         destroy_hist_fields(hist_data);
4439         tracing_map_destroy(hist_data->map);
4440
4441         destroy_actions(hist_data);
4442         destroy_field_vars(hist_data);
4443         destroy_field_var_hists(hist_data);
4444         destroy_synth_var_refs(hist_data);
4445
4446         kfree(hist_data);
4447 }
4448
4449 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4450 {
4451         struct tracing_map *map = hist_data->map;
4452         struct ftrace_event_field *field;
4453         struct hist_field *hist_field;
4454         int i, idx = 0;
4455
4456         for_each_hist_field(i, hist_data) {
4457                 hist_field = hist_data->fields[i];
4458                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4459                         tracing_map_cmp_fn_t cmp_fn;
4460
4461                         field = hist_field->field;
4462
4463                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4464                                 cmp_fn = tracing_map_cmp_none;
4465                         else if (!field)
4466                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4467                                                              hist_field->is_signed);
4468                         else if (is_string_field(field))
4469                                 cmp_fn = tracing_map_cmp_string;
4470                         else
4471                                 cmp_fn = tracing_map_cmp_num(field->size,
4472                                                              field->is_signed);
4473                         idx = tracing_map_add_key_field(map,
4474                                                         hist_field->offset,
4475                                                         cmp_fn);
4476                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4477                         idx = tracing_map_add_sum_field(map);
4478
4479                 if (idx < 0)
4480                         return idx;
4481
4482                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4483                         idx = tracing_map_add_var(map);
4484                         if (idx < 0)
4485                                 return idx;
4486                         hist_field->var.idx = idx;
4487                         hist_field->var.hist_data = hist_data;
4488                 }
4489         }
4490
4491         return 0;
4492 }
4493
4494 static struct hist_trigger_data *
4495 create_hist_data(unsigned int map_bits,
4496                  struct hist_trigger_attrs *attrs,
4497                  struct trace_event_file *file,
4498                  bool remove)
4499 {
4500         const struct tracing_map_ops *map_ops = NULL;
4501         struct hist_trigger_data *hist_data;
4502         int ret = 0;
4503
4504         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4505         if (!hist_data)
4506                 return ERR_PTR(-ENOMEM);
4507
4508         hist_data->attrs = attrs;
4509         hist_data->remove = remove;
4510         hist_data->event_file = file;
4511
4512         ret = parse_actions(hist_data);
4513         if (ret)
4514                 goto free;
4515
4516         ret = create_hist_fields(hist_data, file);
4517         if (ret)
4518                 goto free;
4519
4520         ret = create_sort_keys(hist_data);
4521         if (ret)
4522                 goto free;
4523
4524         map_ops = &hist_trigger_elt_data_ops;
4525
4526         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4527                                             map_ops, hist_data);
4528         if (IS_ERR(hist_data->map)) {
4529                 ret = PTR_ERR(hist_data->map);
4530                 hist_data->map = NULL;
4531                 goto free;
4532         }
4533
4534         ret = create_tracing_map_fields(hist_data);
4535         if (ret)
4536                 goto free;
4537  out:
4538         return hist_data;
4539  free:
4540         hist_data->attrs = NULL;
4541
4542         destroy_hist_data(hist_data);
4543
4544         hist_data = ERR_PTR(ret);
4545
4546         goto out;
4547 }
4548
4549 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4550                                     struct tracing_map_elt *elt, void *rec,
4551                                     struct ring_buffer_event *rbe,
4552                                     u64 *var_ref_vals)
4553 {
4554         struct hist_elt_data *elt_data;
4555         struct hist_field *hist_field;
4556         unsigned int i, var_idx;
4557         u64 hist_val;
4558
4559         elt_data = elt->private_data;
4560         elt_data->var_ref_vals = var_ref_vals;
4561
4562         for_each_hist_val_field(i, hist_data) {
4563                 hist_field = hist_data->fields[i];
4564                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4565                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4566                         var_idx = hist_field->var.idx;
4567                         tracing_map_set_var(elt, var_idx, hist_val);
4568                         continue;
4569                 }
4570                 tracing_map_update_sum(elt, i, hist_val);
4571         }
4572
4573         for_each_hist_key_field(i, hist_data) {
4574                 hist_field = hist_data->fields[i];
4575                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4576                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4577                         var_idx = hist_field->var.idx;
4578                         tracing_map_set_var(elt, var_idx, hist_val);
4579                 }
4580         }
4581
4582         update_field_vars(hist_data, elt, rbe, rec);
4583 }
4584
4585 static inline void add_to_key(char *compound_key, void *key,
4586                               struct hist_field *key_field, void *rec)
4587 {
4588         size_t size = key_field->size;
4589
4590         if (key_field->flags & HIST_FIELD_FL_STRING) {
4591                 struct ftrace_event_field *field;
4592
4593                 field = key_field->field;
4594                 if (field->filter_type == FILTER_DYN_STRING)
4595                         size = *(u32 *)(rec + field->offset) >> 16;
4596                 else if (field->filter_type == FILTER_PTR_STRING)
4597                         size = strlen(key);
4598                 else if (field->filter_type == FILTER_STATIC_STRING)
4599                         size = field->size;
4600
4601                 /* ensure NULL-termination */
4602                 if (size > key_field->size - 1)
4603                         size = key_field->size - 1;
4604         }
4605
4606         memcpy(compound_key + key_field->offset, key, size);
4607 }
4608
4609 static void
4610 hist_trigger_actions(struct hist_trigger_data *hist_data,
4611                      struct tracing_map_elt *elt, void *rec,
4612                      struct ring_buffer_event *rbe, u64 *var_ref_vals)
4613 {
4614         struct action_data *data;
4615         unsigned int i;
4616
4617         for (i = 0; i < hist_data->n_actions; i++) {
4618                 data = hist_data->actions[i];
4619                 data->fn(hist_data, elt, rec, rbe, data, var_ref_vals);
4620         }
4621 }
4622
4623 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4624                                struct ring_buffer_event *rbe)
4625 {
4626         struct hist_trigger_data *hist_data = data->private_data;
4627         bool use_compound_key = (hist_data->n_keys > 1);
4628         unsigned long entries[HIST_STACKTRACE_DEPTH];
4629         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4630         char compound_key[HIST_KEY_SIZE_MAX];
4631         struct tracing_map_elt *elt = NULL;
4632         struct stack_trace stacktrace;
4633         struct hist_field *key_field;
4634         u64 field_contents;
4635         void *key = NULL;
4636         unsigned int i;
4637
4638         memset(compound_key, 0, hist_data->key_size);
4639
4640         for_each_hist_key_field(i, hist_data) {
4641                 key_field = hist_data->fields[i];
4642
4643                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4644                         stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
4645                         stacktrace.entries = entries;
4646                         stacktrace.nr_entries = 0;
4647                         stacktrace.skip = HIST_STACKTRACE_SKIP;
4648
4649                         memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
4650                         save_stack_trace(&stacktrace);
4651
4652                         key = entries;
4653                 } else {
4654                         field_contents = key_field->fn(key_field, elt, rbe, rec);
4655                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4656                                 key = (void *)(unsigned long)field_contents;
4657                                 use_compound_key = true;
4658                         } else
4659                                 key = (void *)&field_contents;
4660                 }
4661
4662                 if (use_compound_key)
4663                         add_to_key(compound_key, key, key_field, rec);
4664         }
4665
4666         if (use_compound_key)
4667                 key = compound_key;
4668
4669         if (hist_data->n_var_refs &&
4670             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4671                 return;
4672
4673         elt = tracing_map_insert(hist_data->map, key);
4674         if (!elt)
4675                 return;
4676
4677         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4678
4679         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4680                 hist_trigger_actions(hist_data, elt, rec, rbe, var_ref_vals);
4681 }
4682
4683 static void hist_trigger_stacktrace_print(struct seq_file *m,
4684                                           unsigned long *stacktrace_entries,
4685                                           unsigned int max_entries)
4686 {
4687         char str[KSYM_SYMBOL_LEN];
4688         unsigned int spaces = 8;
4689         unsigned int i;
4690
4691         for (i = 0; i < max_entries; i++) {
4692                 if (stacktrace_entries[i] == ULONG_MAX)
4693                         return;
4694
4695                 seq_printf(m, "%*c", 1 + spaces, ' ');
4696                 sprint_symbol(str, stacktrace_entries[i]);
4697                 seq_printf(m, "%s\n", str);
4698         }
4699 }
4700
4701 static void
4702 hist_trigger_entry_print(struct seq_file *m,
4703                          struct hist_trigger_data *hist_data, void *key,
4704                          struct tracing_map_elt *elt)
4705 {
4706         struct hist_field *key_field;
4707         char str[KSYM_SYMBOL_LEN];
4708         bool multiline = false;
4709         const char *field_name;
4710         unsigned int i;
4711         u64 uval;
4712
4713         seq_puts(m, "{ ");
4714
4715         for_each_hist_key_field(i, hist_data) {
4716                 key_field = hist_data->fields[i];
4717
4718                 if (i > hist_data->n_vals)
4719                         seq_puts(m, ", ");
4720
4721                 field_name = hist_field_name(key_field, 0);
4722
4723                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4724                         uval = *(u64 *)(key + key_field->offset);
4725                         seq_printf(m, "%s: %llx", field_name, uval);
4726                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4727                         uval = *(u64 *)(key + key_field->offset);
4728                         sprint_symbol_no_offset(str, uval);
4729                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4730                                    uval, str);
4731                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4732                         uval = *(u64 *)(key + key_field->offset);
4733                         sprint_symbol(str, uval);
4734                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4735                                    uval, str);
4736                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4737                         struct hist_elt_data *elt_data = elt->private_data;
4738                         char *comm;
4739
4740                         if (WARN_ON_ONCE(!elt_data))
4741                                 return;
4742
4743                         comm = elt_data->comm;
4744
4745                         uval = *(u64 *)(key + key_field->offset);
4746                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4747                                    comm, uval);
4748                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4749                         const char *syscall_name;
4750
4751                         uval = *(u64 *)(key + key_field->offset);
4752                         syscall_name = get_syscall_name(uval);
4753                         if (!syscall_name)
4754                                 syscall_name = "unknown_syscall";
4755
4756                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4757                                    syscall_name, uval);
4758                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4759                         seq_puts(m, "stacktrace:\n");
4760                         hist_trigger_stacktrace_print(m,
4761                                                       key + key_field->offset,
4762                                                       HIST_STACKTRACE_DEPTH);
4763                         multiline = true;
4764                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4765                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4766                                    *(u64 *)(key + key_field->offset));
4767                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4768                         seq_printf(m, "%s: %-50s", field_name,
4769                                    (char *)(key + key_field->offset));
4770                 } else {
4771                         uval = *(u64 *)(key + key_field->offset);
4772                         seq_printf(m, "%s: %10llu", field_name, uval);
4773                 }
4774         }
4775
4776         if (!multiline)
4777                 seq_puts(m, " ");
4778
4779         seq_puts(m, "}");
4780
4781         seq_printf(m, " hitcount: %10llu",
4782                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4783
4784         for (i = 1; i < hist_data->n_vals; i++) {
4785                 field_name = hist_field_name(hist_data->fields[i], 0);
4786
4787                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4788                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4789                         continue;
4790
4791                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4792                         seq_printf(m, "  %s: %10llx", field_name,
4793                                    tracing_map_read_sum(elt, i));
4794                 } else {
4795                         seq_printf(m, "  %s: %10llu", field_name,
4796                                    tracing_map_read_sum(elt, i));
4797                 }
4798         }
4799
4800         print_actions(m, hist_data, elt);
4801
4802         seq_puts(m, "\n");
4803 }
4804
4805 static int print_entries(struct seq_file *m,
4806                          struct hist_trigger_data *hist_data)
4807 {
4808         struct tracing_map_sort_entry **sort_entries = NULL;
4809         struct tracing_map *map = hist_data->map;
4810         int i, n_entries;
4811
4812         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4813                                              hist_data->n_sort_keys,
4814                                              &sort_entries);
4815         if (n_entries < 0)
4816                 return n_entries;
4817
4818         for (i = 0; i < n_entries; i++)
4819                 hist_trigger_entry_print(m, hist_data,
4820                                          sort_entries[i]->key,
4821                                          sort_entries[i]->elt);
4822
4823         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4824
4825         return n_entries;
4826 }
4827
4828 static void hist_trigger_show(struct seq_file *m,
4829                               struct event_trigger_data *data, int n)
4830 {
4831         struct hist_trigger_data *hist_data;
4832         int n_entries;
4833
4834         if (n > 0)
4835                 seq_puts(m, "\n\n");
4836
4837         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4838         data->ops->print(m, data->ops, data);
4839         seq_puts(m, "#\n\n");
4840
4841         hist_data = data->private_data;
4842         n_entries = print_entries(m, hist_data);
4843         if (n_entries < 0)
4844                 n_entries = 0;
4845
4846         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4847                    (u64)atomic64_read(&hist_data->map->hits),
4848                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4849 }
4850
4851 static int hist_show(struct seq_file *m, void *v)
4852 {
4853         struct event_trigger_data *data;
4854         struct trace_event_file *event_file;
4855         int n = 0, ret = 0;
4856
4857         mutex_lock(&event_mutex);
4858
4859         event_file = event_file_data(m->private);
4860         if (unlikely(!event_file)) {
4861                 ret = -ENODEV;
4862                 goto out_unlock;
4863         }
4864
4865         list_for_each_entry_rcu(data, &event_file->triggers, list) {
4866                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4867                         hist_trigger_show(m, data, n++);
4868         }
4869
4870         if (have_hist_err()) {
4871                 seq_printf(m, "\nERROR: %s\n", hist_err_str);
4872                 seq_printf(m, "  Last command: %s\n", last_hist_cmd);
4873         }
4874
4875  out_unlock:
4876         mutex_unlock(&event_mutex);
4877
4878         return ret;
4879 }
4880
4881 static int event_hist_open(struct inode *inode, struct file *file)
4882 {
4883         return single_open(file, hist_show, file);
4884 }
4885
4886 const struct file_operations event_hist_fops = {
4887         .open = event_hist_open,
4888         .read = seq_read,
4889         .llseek = seq_lseek,
4890         .release = single_release,
4891 };
4892
4893 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
4894 {
4895         const char *field_name = hist_field_name(hist_field, 0);
4896
4897         if (hist_field->var.name)
4898                 seq_printf(m, "%s=", hist_field->var.name);
4899
4900         if (hist_field->flags & HIST_FIELD_FL_CPU)
4901                 seq_puts(m, "cpu");
4902         else if (field_name) {
4903                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
4904                     hist_field->flags & HIST_FIELD_FL_ALIAS)
4905                         seq_putc(m, '$');
4906                 seq_printf(m, "%s", field_name);
4907         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
4908                 seq_puts(m, "common_timestamp");
4909
4910         if (hist_field->flags) {
4911                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
4912                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
4913                         const char *flags = get_hist_field_flags(hist_field);
4914
4915                         if (flags)
4916                                 seq_printf(m, ".%s", flags);
4917                 }
4918         }
4919 }
4920
4921 static int event_hist_trigger_print(struct seq_file *m,
4922                                     struct event_trigger_ops *ops,
4923                                     struct event_trigger_data *data)
4924 {
4925         struct hist_trigger_data *hist_data = data->private_data;
4926         struct hist_field *field;
4927         bool have_var = false;
4928         unsigned int i;
4929
4930         seq_puts(m, "hist:");
4931
4932         if (data->name)
4933                 seq_printf(m, "%s:", data->name);
4934
4935         seq_puts(m, "keys=");
4936
4937         for_each_hist_key_field(i, hist_data) {
4938                 field = hist_data->fields[i];
4939
4940                 if (i > hist_data->n_vals)
4941                         seq_puts(m, ",");
4942
4943                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
4944                         seq_puts(m, "stacktrace");
4945                 else
4946                         hist_field_print(m, field);
4947         }
4948
4949         seq_puts(m, ":vals=");
4950
4951         for_each_hist_val_field(i, hist_data) {
4952                 field = hist_data->fields[i];
4953                 if (field->flags & HIST_FIELD_FL_VAR) {
4954                         have_var = true;
4955                         continue;
4956                 }
4957
4958                 if (i == HITCOUNT_IDX)
4959                         seq_puts(m, "hitcount");
4960                 else {
4961                         seq_puts(m, ",");
4962                         hist_field_print(m, field);
4963                 }
4964         }
4965
4966         if (have_var) {
4967                 unsigned int n = 0;
4968
4969                 seq_puts(m, ":");
4970
4971                 for_each_hist_val_field(i, hist_data) {
4972                         field = hist_data->fields[i];
4973
4974                         if (field->flags & HIST_FIELD_FL_VAR) {
4975                                 if (n++)
4976                                         seq_puts(m, ",");
4977                                 hist_field_print(m, field);
4978                         }
4979                 }
4980         }
4981
4982         seq_puts(m, ":sort=");
4983
4984         for (i = 0; i < hist_data->n_sort_keys; i++) {
4985                 struct tracing_map_sort_key *sort_key;
4986                 unsigned int idx, first_key_idx;
4987
4988                 /* skip VAR vals */
4989                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
4990
4991                 sort_key = &hist_data->sort_keys[i];
4992                 idx = sort_key->field_idx;
4993
4994                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
4995                         return -EINVAL;
4996
4997                 if (i > 0)
4998                         seq_puts(m, ",");
4999
5000                 if (idx == HITCOUNT_IDX)
5001                         seq_puts(m, "hitcount");
5002                 else {
5003                         if (idx >= first_key_idx)
5004                                 idx += hist_data->n_vars;
5005                         hist_field_print(m, hist_data->fields[idx]);
5006                 }
5007
5008                 if (sort_key->descending)
5009                         seq_puts(m, ".descending");
5010         }
5011         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5012         if (hist_data->enable_timestamps)
5013                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5014
5015         print_actions_spec(m, hist_data);
5016
5017         if (data->filter_str)
5018                 seq_printf(m, " if %s", data->filter_str);
5019
5020         if (data->paused)
5021                 seq_puts(m, " [paused]");
5022         else
5023                 seq_puts(m, " [active]");
5024
5025         seq_putc(m, '\n');
5026
5027         return 0;
5028 }
5029
5030 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5031                                    struct event_trigger_data *data)
5032 {
5033         struct hist_trigger_data *hist_data = data->private_data;
5034
5035         if (!data->ref && hist_data->attrs->name)
5036                 save_named_trigger(hist_data->attrs->name, data);
5037
5038         data->ref++;
5039
5040         return 0;
5041 }
5042
5043 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5044 {
5045         struct trace_event_file *file;
5046         unsigned int i;
5047         char *cmd;
5048         int ret;
5049
5050         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5051                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5052                 cmd = hist_data->field_var_hists[i]->cmd;
5053                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5054                                               "!hist", "hist", cmd);
5055         }
5056 }
5057
5058 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5059                                     struct event_trigger_data *data)
5060 {
5061         struct hist_trigger_data *hist_data = data->private_data;
5062
5063         if (WARN_ON_ONCE(data->ref <= 0))
5064                 return;
5065
5066         data->ref--;
5067         if (!data->ref) {
5068                 if (data->name)
5069                         del_named_trigger(data);
5070
5071                 trigger_data_free(data);
5072
5073                 remove_hist_vars(hist_data);
5074
5075                 unregister_field_var_hists(hist_data);
5076
5077                 destroy_hist_data(hist_data);
5078         }
5079 }
5080
5081 static struct event_trigger_ops event_hist_trigger_ops = {
5082         .func                   = event_hist_trigger,
5083         .print                  = event_hist_trigger_print,
5084         .init                   = event_hist_trigger_init,
5085         .free                   = event_hist_trigger_free,
5086 };
5087
5088 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5089                                          struct event_trigger_data *data)
5090 {
5091         data->ref++;
5092
5093         save_named_trigger(data->named_data->name, data);
5094
5095         event_hist_trigger_init(ops, data->named_data);
5096
5097         return 0;
5098 }
5099
5100 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5101                                           struct event_trigger_data *data)
5102 {
5103         if (WARN_ON_ONCE(data->ref <= 0))
5104                 return;
5105
5106         event_hist_trigger_free(ops, data->named_data);
5107
5108         data->ref--;
5109         if (!data->ref) {
5110                 del_named_trigger(data);
5111                 trigger_data_free(data);
5112         }
5113 }
5114
5115 static struct event_trigger_ops event_hist_trigger_named_ops = {
5116         .func                   = event_hist_trigger,
5117         .print                  = event_hist_trigger_print,
5118         .init                   = event_hist_trigger_named_init,
5119         .free                   = event_hist_trigger_named_free,
5120 };
5121
5122 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5123                                                             char *param)
5124 {
5125         return &event_hist_trigger_ops;
5126 }
5127
5128 static void hist_clear(struct event_trigger_data *data)
5129 {
5130         struct hist_trigger_data *hist_data = data->private_data;
5131
5132         if (data->name)
5133                 pause_named_trigger(data);
5134
5135         tracepoint_synchronize_unregister();
5136
5137         tracing_map_clear(hist_data->map);
5138
5139         if (data->name)
5140                 unpause_named_trigger(data);
5141 }
5142
5143 static bool compatible_field(struct ftrace_event_field *field,
5144                              struct ftrace_event_field *test_field)
5145 {
5146         if (field == test_field)
5147                 return true;
5148         if (field == NULL || test_field == NULL)
5149                 return false;
5150         if (strcmp(field->name, test_field->name) != 0)
5151                 return false;
5152         if (strcmp(field->type, test_field->type) != 0)
5153                 return false;
5154         if (field->size != test_field->size)
5155                 return false;
5156         if (field->is_signed != test_field->is_signed)
5157                 return false;
5158
5159         return true;
5160 }
5161
5162 static bool hist_trigger_match(struct event_trigger_data *data,
5163                                struct event_trigger_data *data_test,
5164                                struct event_trigger_data *named_data,
5165                                bool ignore_filter)
5166 {
5167         struct tracing_map_sort_key *sort_key, *sort_key_test;
5168         struct hist_trigger_data *hist_data, *hist_data_test;
5169         struct hist_field *key_field, *key_field_test;
5170         unsigned int i;
5171
5172         if (named_data && (named_data != data_test) &&
5173             (named_data != data_test->named_data))
5174                 return false;
5175
5176         if (!named_data && is_named_trigger(data_test))
5177                 return false;
5178
5179         hist_data = data->private_data;
5180         hist_data_test = data_test->private_data;
5181
5182         if (hist_data->n_vals != hist_data_test->n_vals ||
5183             hist_data->n_fields != hist_data_test->n_fields ||
5184             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5185                 return false;
5186
5187         if (!ignore_filter) {
5188                 if ((data->filter_str && !data_test->filter_str) ||
5189                    (!data->filter_str && data_test->filter_str))
5190                         return false;
5191         }
5192
5193         for_each_hist_field(i, hist_data) {
5194                 key_field = hist_data->fields[i];
5195                 key_field_test = hist_data_test->fields[i];
5196
5197                 if (key_field->flags != key_field_test->flags)
5198                         return false;
5199                 if (!compatible_field(key_field->field, key_field_test->field))
5200                         return false;
5201                 if (key_field->offset != key_field_test->offset)
5202                         return false;
5203                 if (key_field->size != key_field_test->size)
5204                         return false;
5205                 if (key_field->is_signed != key_field_test->is_signed)
5206                         return false;
5207                 if (!!key_field->var.name != !!key_field_test->var.name)
5208                         return false;
5209                 if (key_field->var.name &&
5210                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5211                         return false;
5212         }
5213
5214         for (i = 0; i < hist_data->n_sort_keys; i++) {
5215                 sort_key = &hist_data->sort_keys[i];
5216                 sort_key_test = &hist_data_test->sort_keys[i];
5217
5218                 if (sort_key->field_idx != sort_key_test->field_idx ||
5219                     sort_key->descending != sort_key_test->descending)
5220                         return false;
5221         }
5222
5223         if (!ignore_filter && data->filter_str &&
5224             (strcmp(data->filter_str, data_test->filter_str) != 0))
5225                 return false;
5226
5227         if (!actions_match(hist_data, hist_data_test))
5228                 return false;
5229
5230         return true;
5231 }
5232
5233 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5234                                  struct event_trigger_data *data,
5235                                  struct trace_event_file *file)
5236 {
5237         struct hist_trigger_data *hist_data = data->private_data;
5238         struct event_trigger_data *test, *named_data = NULL;
5239         int ret = 0;
5240
5241         if (hist_data->attrs->name) {
5242                 named_data = find_named_trigger(hist_data->attrs->name);
5243                 if (named_data) {
5244                         if (!hist_trigger_match(data, named_data, named_data,
5245                                                 true)) {
5246                                 hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name);
5247                                 ret = -EINVAL;
5248                                 goto out;
5249                         }
5250                 }
5251         }
5252
5253         if (hist_data->attrs->name && !named_data)
5254                 goto new;
5255
5256         list_for_each_entry_rcu(test, &file->triggers, list) {
5257                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5258                         if (!hist_trigger_match(data, test, named_data, false))
5259                                 continue;
5260                         if (hist_data->attrs->pause)
5261                                 test->paused = true;
5262                         else if (hist_data->attrs->cont)
5263                                 test->paused = false;
5264                         else if (hist_data->attrs->clear)
5265                                 hist_clear(test);
5266                         else {
5267                                 hist_err("Hist trigger already exists", NULL);
5268                                 ret = -EEXIST;
5269                         }
5270                         goto out;
5271                 }
5272         }
5273  new:
5274         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5275                 hist_err("Can't clear or continue a nonexistent hist trigger", NULL);
5276                 ret = -ENOENT;
5277                 goto out;
5278         }
5279
5280         if (hist_data->attrs->pause)
5281                 data->paused = true;
5282
5283         if (named_data) {
5284                 data->private_data = named_data->private_data;
5285                 set_named_trigger_data(data, named_data);
5286                 data->ops = &event_hist_trigger_named_ops;
5287         }
5288
5289         if (data->ops->init) {
5290                 ret = data->ops->init(data->ops, data);
5291                 if (ret < 0)
5292                         goto out;
5293         }
5294
5295         if (hist_data->enable_timestamps) {
5296                 char *clock = hist_data->attrs->clock;
5297
5298                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5299                 if (ret) {
5300                         hist_err("Couldn't set trace_clock: ", clock);
5301                         goto out;
5302                 }
5303
5304                 tracing_set_time_stamp_abs(file->tr, true);
5305         }
5306
5307         if (named_data)
5308                 destroy_hist_data(hist_data);
5309
5310         ret++;
5311  out:
5312         return ret;
5313 }
5314
5315 static int hist_trigger_enable(struct event_trigger_data *data,
5316                                struct trace_event_file *file)
5317 {
5318         int ret = 0;
5319
5320         list_add_tail_rcu(&data->list, &file->triggers);
5321
5322         update_cond_flag(file);
5323
5324         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5325                 list_del_rcu(&data->list);
5326                 update_cond_flag(file);
5327                 ret--;
5328         }
5329
5330         return ret;
5331 }
5332
5333 static bool have_hist_trigger_match(struct event_trigger_data *data,
5334                                     struct trace_event_file *file)
5335 {
5336         struct hist_trigger_data *hist_data = data->private_data;
5337         struct event_trigger_data *test, *named_data = NULL;
5338         bool match = false;
5339
5340         if (hist_data->attrs->name)
5341                 named_data = find_named_trigger(hist_data->attrs->name);
5342
5343         list_for_each_entry_rcu(test, &file->triggers, list) {
5344                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5345                         if (hist_trigger_match(data, test, named_data, false)) {
5346                                 match = true;
5347                                 break;
5348                         }
5349                 }
5350         }
5351
5352         return match;
5353 }
5354
5355 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5356                                     struct trace_event_file *file)
5357 {
5358         struct hist_trigger_data *hist_data = data->private_data;
5359         struct event_trigger_data *test, *named_data = NULL;
5360
5361         if (hist_data->attrs->name)
5362                 named_data = find_named_trigger(hist_data->attrs->name);
5363
5364         list_for_each_entry_rcu(test, &file->triggers, list) {
5365                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5366                         if (!hist_trigger_match(data, test, named_data, false))
5367                                 continue;
5368                         hist_data = test->private_data;
5369                         if (check_var_refs(hist_data))
5370                                 return true;
5371                         break;
5372                 }
5373         }
5374
5375         return false;
5376 }
5377
5378 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5379                                     struct event_trigger_data *data,
5380                                     struct trace_event_file *file)
5381 {
5382         struct hist_trigger_data *hist_data = data->private_data;
5383         struct event_trigger_data *test, *named_data = NULL;
5384         bool unregistered = false;
5385
5386         if (hist_data->attrs->name)
5387                 named_data = find_named_trigger(hist_data->attrs->name);
5388
5389         list_for_each_entry_rcu(test, &file->triggers, list) {
5390                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5391                         if (!hist_trigger_match(data, test, named_data, false))
5392                                 continue;
5393                         unregistered = true;
5394                         list_del_rcu(&test->list);
5395                         trace_event_trigger_enable_disable(file, 0);
5396                         update_cond_flag(file);
5397                         break;
5398                 }
5399         }
5400
5401         if (unregistered && test->ops->free)
5402                 test->ops->free(test->ops, test);
5403
5404         if (hist_data->enable_timestamps) {
5405                 if (!hist_data->remove || unregistered)
5406                         tracing_set_time_stamp_abs(file->tr, false);
5407         }
5408 }
5409
5410 static bool hist_file_check_refs(struct trace_event_file *file)
5411 {
5412         struct hist_trigger_data *hist_data;
5413         struct event_trigger_data *test;
5414
5415         list_for_each_entry_rcu(test, &file->triggers, list) {
5416                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5417                         hist_data = test->private_data;
5418                         if (check_var_refs(hist_data))
5419                                 return true;
5420                 }
5421         }
5422
5423         return false;
5424 }
5425
5426 static void hist_unreg_all(struct trace_event_file *file)
5427 {
5428         struct event_trigger_data *test, *n;
5429         struct hist_trigger_data *hist_data;
5430         struct synth_event *se;
5431         const char *se_name;
5432
5433         if (hist_file_check_refs(file))
5434                 return;
5435
5436         list_for_each_entry_safe(test, n, &file->triggers, list) {
5437                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5438                         hist_data = test->private_data;
5439                         list_del_rcu(&test->list);
5440                         trace_event_trigger_enable_disable(file, 0);
5441
5442                         mutex_lock(&synth_event_mutex);
5443                         se_name = trace_event_name(file->event_call);
5444                         se = find_synth_event(se_name);
5445                         if (se)
5446                                 se->ref--;
5447                         mutex_unlock(&synth_event_mutex);
5448
5449                         update_cond_flag(file);
5450                         if (hist_data->enable_timestamps)
5451                                 tracing_set_time_stamp_abs(file->tr, false);
5452                         if (test->ops->free)
5453                                 test->ops->free(test->ops, test);
5454                 }
5455         }
5456 }
5457
5458 static int event_hist_trigger_func(struct event_command *cmd_ops,
5459                                    struct trace_event_file *file,
5460                                    char *glob, char *cmd, char *param)
5461 {
5462         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5463         struct event_trigger_data *trigger_data;
5464         struct hist_trigger_attrs *attrs;
5465         struct event_trigger_ops *trigger_ops;
5466         struct hist_trigger_data *hist_data;
5467         struct synth_event *se;
5468         const char *se_name;
5469         bool remove = false;
5470         char *trigger, *p;
5471         int ret = 0;
5472
5473         if (glob && strlen(glob)) {
5474                 last_cmd_set(param);
5475                 hist_err_clear();
5476         }
5477
5478         if (!param)
5479                 return -EINVAL;
5480
5481         if (glob[0] == '!')
5482                 remove = true;
5483
5484         /*
5485          * separate the trigger from the filter (k:v [if filter])
5486          * allowing for whitespace in the trigger
5487          */
5488         p = trigger = param;
5489         do {
5490                 p = strstr(p, "if");
5491                 if (!p)
5492                         break;
5493                 if (p == param)
5494                         return -EINVAL;
5495                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5496                         p++;
5497                         continue;
5498                 }
5499                 if (p >= param + strlen(param) - strlen("if") - 1)
5500                         return -EINVAL;
5501                 if (*(p + strlen("if")) != ' ' && *(p + strlen("if")) != '\t') {
5502                         p++;
5503                         continue;
5504                 }
5505                 break;
5506         } while (p);
5507
5508         if (!p)
5509                 param = NULL;
5510         else {
5511                 *(p - 1) = '\0';
5512                 param = strstrip(p);
5513                 trigger = strstrip(trigger);
5514         }
5515
5516         attrs = parse_hist_trigger_attrs(trigger);
5517         if (IS_ERR(attrs))
5518                 return PTR_ERR(attrs);
5519
5520         if (attrs->map_bits)
5521                 hist_trigger_bits = attrs->map_bits;
5522
5523         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5524         if (IS_ERR(hist_data)) {
5525                 destroy_hist_trigger_attrs(attrs);
5526                 return PTR_ERR(hist_data);
5527         }
5528
5529         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5530
5531         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5532         if (!trigger_data) {
5533                 ret = -ENOMEM;
5534                 goto out_free;
5535         }
5536
5537         trigger_data->count = -1;
5538         trigger_data->ops = trigger_ops;
5539         trigger_data->cmd_ops = cmd_ops;
5540
5541         INIT_LIST_HEAD(&trigger_data->list);
5542         RCU_INIT_POINTER(trigger_data->filter, NULL);
5543
5544         trigger_data->private_data = hist_data;
5545
5546         /* if param is non-empty, it's supposed to be a filter */
5547         if (param && cmd_ops->set_filter) {
5548                 ret = cmd_ops->set_filter(param, trigger_data, file);
5549                 if (ret < 0)
5550                         goto out_free;
5551         }
5552
5553         if (remove) {
5554                 if (!have_hist_trigger_match(trigger_data, file))
5555                         goto out_free;
5556
5557                 if (hist_trigger_check_refs(trigger_data, file)) {
5558                         ret = -EBUSY;
5559                         goto out_free;
5560                 }
5561
5562                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5563
5564                 mutex_lock(&synth_event_mutex);
5565                 se_name = trace_event_name(file->event_call);
5566                 se = find_synth_event(se_name);
5567                 if (se)
5568                         se->ref--;
5569                 mutex_unlock(&synth_event_mutex);
5570
5571                 ret = 0;
5572                 goto out_free;
5573         }
5574
5575         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5576         /*
5577          * The above returns on success the # of triggers registered,
5578          * but if it didn't register any it returns zero.  Consider no
5579          * triggers registered a failure too.
5580          */
5581         if (!ret) {
5582                 if (!(attrs->pause || attrs->cont || attrs->clear))
5583                         ret = -ENOENT;
5584                 goto out_free;
5585         } else if (ret < 0)
5586                 goto out_free;
5587
5588         if (get_named_trigger_data(trigger_data))
5589                 goto enable;
5590
5591         if (has_hist_vars(hist_data))
5592                 save_hist_vars(hist_data);
5593
5594         ret = create_actions(hist_data, file);
5595         if (ret)
5596                 goto out_unreg;
5597
5598         ret = tracing_map_init(hist_data->map);
5599         if (ret)
5600                 goto out_unreg;
5601 enable:
5602         ret = hist_trigger_enable(trigger_data, file);
5603         if (ret)
5604                 goto out_unreg;
5605
5606         mutex_lock(&synth_event_mutex);
5607         se_name = trace_event_name(file->event_call);
5608         se = find_synth_event(se_name);
5609         if (se)
5610                 se->ref++;
5611         mutex_unlock(&synth_event_mutex);
5612
5613         /* Just return zero, not the number of registered triggers */
5614         ret = 0;
5615  out:
5616         if (ret == 0)
5617                 hist_err_clear();
5618
5619         return ret;
5620  out_unreg:
5621         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5622  out_free:
5623         if (cmd_ops->set_filter)
5624                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5625
5626         remove_hist_vars(hist_data);
5627
5628         kfree(trigger_data);
5629
5630         destroy_hist_data(hist_data);
5631         goto out;
5632 }
5633
5634 static struct event_command trigger_hist_cmd = {
5635         .name                   = "hist",
5636         .trigger_type           = ETT_EVENT_HIST,
5637         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5638         .func                   = event_hist_trigger_func,
5639         .reg                    = hist_register_trigger,
5640         .unreg                  = hist_unregister_trigger,
5641         .unreg_all              = hist_unreg_all,
5642         .get_trigger_ops        = event_hist_get_trigger_ops,
5643         .set_filter             = set_trigger_filter,
5644 };
5645
5646 __init int register_trigger_hist_cmd(void)
5647 {
5648         int ret;
5649
5650         ret = register_event_command(&trigger_hist_cmd);
5651         WARN_ON(ret < 0);
5652
5653         return ret;
5654 }
5655
5656 static void
5657 hist_enable_trigger(struct event_trigger_data *data, void *rec,
5658                     struct ring_buffer_event *event)
5659 {
5660         struct enable_trigger_data *enable_data = data->private_data;
5661         struct event_trigger_data *test;
5662
5663         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
5664                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5665                         if (enable_data->enable)
5666                                 test->paused = false;
5667                         else
5668                                 test->paused = true;
5669                 }
5670         }
5671 }
5672
5673 static void
5674 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5675                           struct ring_buffer_event *event)
5676 {
5677         if (!data->count)
5678                 return;
5679
5680         if (data->count != -1)
5681                 (data->count)--;
5682
5683         hist_enable_trigger(data, rec, event);
5684 }
5685
5686 static struct event_trigger_ops hist_enable_trigger_ops = {
5687         .func                   = hist_enable_trigger,
5688         .print                  = event_enable_trigger_print,
5689         .init                   = event_trigger_init,
5690         .free                   = event_enable_trigger_free,
5691 };
5692
5693 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5694         .func                   = hist_enable_count_trigger,
5695         .print                  = event_enable_trigger_print,
5696         .init                   = event_trigger_init,
5697         .free                   = event_enable_trigger_free,
5698 };
5699
5700 static struct event_trigger_ops hist_disable_trigger_ops = {
5701         .func                   = hist_enable_trigger,
5702         .print                  = event_enable_trigger_print,
5703         .init                   = event_trigger_init,
5704         .free                   = event_enable_trigger_free,
5705 };
5706
5707 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5708         .func                   = hist_enable_count_trigger,
5709         .print                  = event_enable_trigger_print,
5710         .init                   = event_trigger_init,
5711         .free                   = event_enable_trigger_free,
5712 };
5713
5714 static struct event_trigger_ops *
5715 hist_enable_get_trigger_ops(char *cmd, char *param)
5716 {
5717         struct event_trigger_ops *ops;
5718         bool enable;
5719
5720         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5721
5722         if (enable)
5723                 ops = param ? &hist_enable_count_trigger_ops :
5724                         &hist_enable_trigger_ops;
5725         else
5726                 ops = param ? &hist_disable_count_trigger_ops :
5727                         &hist_disable_trigger_ops;
5728
5729         return ops;
5730 }
5731
5732 static void hist_enable_unreg_all(struct trace_event_file *file)
5733 {
5734         struct event_trigger_data *test, *n;
5735
5736         list_for_each_entry_safe(test, n, &file->triggers, list) {
5737                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5738                         list_del_rcu(&test->list);
5739                         update_cond_flag(file);
5740                         trace_event_trigger_enable_disable(file, 0);
5741                         if (test->ops->free)
5742                                 test->ops->free(test->ops, test);
5743                 }
5744         }
5745 }
5746
5747 static struct event_command trigger_hist_enable_cmd = {
5748         .name                   = ENABLE_HIST_STR,
5749         .trigger_type           = ETT_HIST_ENABLE,
5750         .func                   = event_enable_trigger_func,
5751         .reg                    = event_enable_register_trigger,
5752         .unreg                  = event_enable_unregister_trigger,
5753         .unreg_all              = hist_enable_unreg_all,
5754         .get_trigger_ops        = hist_enable_get_trigger_ops,
5755         .set_filter             = set_trigger_filter,
5756 };
5757
5758 static struct event_command trigger_hist_disable_cmd = {
5759         .name                   = DISABLE_HIST_STR,
5760         .trigger_type           = ETT_HIST_ENABLE,
5761         .func                   = event_enable_trigger_func,
5762         .reg                    = event_enable_register_trigger,
5763         .unreg                  = event_enable_unregister_trigger,
5764         .unreg_all              = hist_enable_unreg_all,
5765         .get_trigger_ops        = hist_enable_get_trigger_ops,
5766         .set_filter             = set_trigger_filter,
5767 };
5768
5769 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5770 {
5771         unregister_event_command(&trigger_hist_enable_cmd);
5772         unregister_event_command(&trigger_hist_disable_cmd);
5773 }
5774
5775 __init int register_trigger_hist_enable_disable_cmds(void)
5776 {
5777         int ret;
5778
5779         ret = register_event_command(&trigger_hist_enable_cmd);
5780         if (WARN_ON(ret < 0))
5781                 return ret;
5782         ret = register_event_command(&trigger_hist_disable_cmd);
5783         if (WARN_ON(ret < 0))
5784                 unregister_trigger_hist_enable_disable_cmds();
5785
5786         return ret;
5787 }
5788
5789 static __init int trace_events_hist_init(void)
5790 {
5791         struct dentry *entry = NULL;
5792         struct dentry *d_tracer;
5793         int err = 0;
5794
5795         d_tracer = tracing_init_dentry();
5796         if (IS_ERR(d_tracer)) {
5797                 err = PTR_ERR(d_tracer);
5798                 goto err;
5799         }
5800
5801         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
5802                                     NULL, &synth_events_fops);
5803         if (!entry) {
5804                 err = -ENODEV;
5805                 goto err;
5806         }
5807
5808         return err;
5809  err:
5810         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
5811
5812         return err;
5813 }
5814
5815 fs_initcall(trace_events_hist_init);