kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/mutex.h>
  11 #include <linux/slab.h>
  12 #include <linux/stacktrace.h>
  13 #include <linux/rculist.h>
  14 #include <linux/tracefs.h>
  15
  16 #include "tracing_map.h"
  17 #include "trace.h"
  18 #include "trace_dynevent.h"
  19
  20 #define SYNTH_SYSTEM            "synthetic"
  21 #define SYNTH_FIELDS_MAX        16
  22
  23 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  24
  25 #define ERRORS                                                          \
  26         C(NONE,                 "No error"),                            \
  27         C(DUPLICATE_VAR,        "Variable already defined"),            \
  28         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  29         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  30         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  31         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  32         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  33         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  34         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  35         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  36         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  37         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  38         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  39         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  40         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  41         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  42         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  43         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  44         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  45         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  46         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  47         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  48         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  49         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  50         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  51         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  52         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  53         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  54         C(INVALID_PARAM,        "Invalid action param"),                \
  55         C(ACTION_NOT_FOUND,     "No action found"),                     \
  56         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  57         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  58         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  59         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  60         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  61         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  62         C(INVALID_REF_KEY,      "Using variable references as keys not supported"), \
  63         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  64         C(FIELD_NOT_FOUND,      "Couldn't find field"),
  65
  66 #undef C
  67 #define C(a, b)         HIST_ERR_##a
  68
  69 enum { ERRORS };
  70
  71 #undef C
  72 #define C(a, b)         b
  73
  74 static const char *err_text[] = { ERRORS };
  75
  76 struct hist_field;
  77
  78 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  79                                 struct tracing_map_elt *elt,
  80                                 struct ring_buffer_event *rbe,
  81                                 void *event);
  82
  83 #define HIST_FIELD_OPERANDS_MAX 2
  84 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  85 #define HIST_ACTIONS_MAX        8
  86
  87 enum field_op_id {
  88         FIELD_OP_NONE,
  89         FIELD_OP_PLUS,
  90         FIELD_OP_MINUS,
  91         FIELD_OP_UNARY_MINUS,
  92 };
  93
  94 /*
  95  * A hist_var (histogram variable) contains variable information for
  96  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
  97  * flag set.  A hist_var has a variable name e.g. ts0, and is
  98  * associated with a given histogram trigger, as specified by
  99  * hist_data.  The hist_var idx is the unique index assigned to the
 100  * variable by the hist trigger's tracing_map.  The idx is what is
 101  * used to set a variable's value and, by a variable reference, to
 102  * retrieve it.
 103  */
 104 struct hist_var {
 105         char                            *name;
 106         struct hist_trigger_data        *hist_data;
 107         unsigned int                    idx;
 108 };
 109
 110 struct hist_field {
 111         struct ftrace_event_field       *field;
 112         unsigned long                   flags;
 113         hist_field_fn_t                 fn;
 114         unsigned int                    size;
 115         unsigned int                    offset;
 116         unsigned int                    is_signed;
 117         const char                      *type;
 118         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 119         struct hist_trigger_data        *hist_data;
 120
 121         /*
 122          * Variable fields contain variable-specific info in var.
 123          */
 124         struct hist_var                 var;
 125         enum field_op_id                operator;
 126         char                            *system;
 127         char                            *event_name;
 128
 129         /*
 130          * The name field is used for EXPR and VAR_REF fields.  VAR
 131          * fields contain the variable name in var.name.
 132          */
 133         char                            *name;
 134
 135         /*
 136          * When a histogram trigger is hit, if it has any references
 137          * to variables, the values of those variables are collected
 138          * into a var_ref_vals array by resolve_var_refs().  The
 139          * current value of each variable is read from the tracing_map
 140          * using the hist field's hist_var.idx and entered into the
 141          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 142          */
 143         unsigned int                    var_ref_idx;
 144         bool                            read_once;
 145 };
 146
 147 static u64 hist_field_none(struct hist_field *field,
 148                            struct tracing_map_elt *elt,
 149                            struct ring_buffer_event *rbe,
 150                            void *event)
 151 {
 152         return 0;
 153 }
 154
 155 static u64 hist_field_counter(struct hist_field *field,
 156                               struct tracing_map_elt *elt,
 157                               struct ring_buffer_event *rbe,
 158                               void *event)
 159 {
 160         return 1;
 161 }
 162
 163 static u64 hist_field_string(struct hist_field *hist_field,
 164                              struct tracing_map_elt *elt,
 165                              struct ring_buffer_event *rbe,
 166                              void *event)
 167 {
 168         char *addr = (char *)(event + hist_field->field->offset);
 169
 170         return (u64)(unsigned long)addr;
 171 }
 172
 173 static u64 hist_field_dynstring(struct hist_field *hist_field,
 174                                 struct tracing_map_elt *elt,
 175                                 struct ring_buffer_event *rbe,
 176                                 void *event)
 177 {
 178         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 179         int str_loc = str_item & 0xffff;
 180         char *addr = (char *)(event + str_loc);
 181
 182         return (u64)(unsigned long)addr;
 183 }
 184
 185 static u64 hist_field_pstring(struct hist_field *hist_field,
 186                               struct tracing_map_elt *elt,
 187                               struct ring_buffer_event *rbe,
 188                               void *event)
 189 {
 190         char **addr = (char **)(event + hist_field->field->offset);
 191
 192         return (u64)(unsigned long)*addr;
 193 }
 194
 195 static u64 hist_field_log2(struct hist_field *hist_field,
 196                            struct tracing_map_elt *elt,
 197                            struct ring_buffer_event *rbe,
 198                            void *event)
 199 {
 200         struct hist_field *operand = hist_field->operands[0];
 201
 202         u64 val = operand->fn(operand, elt, rbe, event);
 203
 204         return (u64) ilog2(roundup_pow_of_two(val));
 205 }
 206
 207 static u64 hist_field_plus(struct hist_field *hist_field,
 208                            struct tracing_map_elt *elt,
 209                            struct ring_buffer_event *rbe,
 210                            void *event)
 211 {
 212         struct hist_field *operand1 = hist_field->operands[0];
 213         struct hist_field *operand2 = hist_field->operands[1];
 214
 215         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 216         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 217
 218         return val1 + val2;
 219 }
 220
 221 static u64 hist_field_minus(struct hist_field *hist_field,
 222                             struct tracing_map_elt *elt,
 223                             struct ring_buffer_event *rbe,
 224                             void *event)
 225 {
 226         struct hist_field *operand1 = hist_field->operands[0];
 227         struct hist_field *operand2 = hist_field->operands[1];
 228
 229         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 230         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 231
 232         return val1 - val2;
 233 }
 234
 235 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 236                                   struct tracing_map_elt *elt,
 237                                   struct ring_buffer_event *rbe,
 238                                   void *event)
 239 {
 240         struct hist_field *operand = hist_field->operands[0];
 241
 242         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 243         u64 val = (u64)-sval;
 244
 245         return val;
 246 }
 247
 248 #define DEFINE_HIST_FIELD_FN(type)                                      \
 249         static u64 hist_field_##type(struct hist_field *hist_field,     \
 250                                      struct tracing_map_elt *elt,       \
 251                                      struct ring_buffer_event *rbe,     \
 252                                      void *event)                       \
 253 {                                                                       \
 254         type *addr = (type *)(event + hist_field->field->offset);       \
 255                                                                         \
 256         return (u64)(unsigned long)*addr;                               \
 257 }
 258
 259 DEFINE_HIST_FIELD_FN(s64);
 260 DEFINE_HIST_FIELD_FN(u64);
 261 DEFINE_HIST_FIELD_FN(s32);
 262 DEFINE_HIST_FIELD_FN(u32);
 263 DEFINE_HIST_FIELD_FN(s16);
 264 DEFINE_HIST_FIELD_FN(u16);
 265 DEFINE_HIST_FIELD_FN(s8);
 266 DEFINE_HIST_FIELD_FN(u8);
 267
 268 #define for_each_hist_field(i, hist_data)       \
 269         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 270
 271 #define for_each_hist_val_field(i, hist_data)   \
 272         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 273
 274 #define for_each_hist_key_field(i, hist_data)   \
 275         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 276
 277 #define HIST_STACKTRACE_DEPTH   16
 278 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 279 #define HIST_STACKTRACE_SKIP    5
 280
 281 #define HITCOUNT_IDX            0
 282 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 283
 284 enum hist_field_flags {
 285         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 286         HIST_FIELD_FL_KEY               = 1 << 1,
 287         HIST_FIELD_FL_STRING            = 1 << 2,
 288         HIST_FIELD_FL_HEX               = 1 << 3,
 289         HIST_FIELD_FL_SYM               = 1 << 4,
 290         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 291         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 292         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 293         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 294         HIST_FIELD_FL_LOG2              = 1 << 9,
 295         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 296         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 297         HIST_FIELD_FL_VAR               = 1 << 12,
 298         HIST_FIELD_FL_EXPR              = 1 << 13,
 299         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 300         HIST_FIELD_FL_CPU               = 1 << 15,
 301         HIST_FIELD_FL_ALIAS             = 1 << 16,
 302 };
 303
 304 struct var_defs {
 305         unsigned int    n_vars;
 306         char            *name[TRACING_MAP_VARS_MAX];
 307         char            *expr[TRACING_MAP_VARS_MAX];
 308 };
 309
 310 struct hist_trigger_attrs {
 311         char            *keys_str;
 312         char            *vals_str;
 313         char            *sort_key_str;
 314         char            *name;
 315         char            *clock;
 316         bool            pause;
 317         bool            cont;
 318         bool            clear;
 319         bool            ts_in_usecs;
 320         unsigned int    map_bits;
 321
 322         char            *assignment_str[TRACING_MAP_VARS_MAX];
 323         unsigned int    n_assignments;
 324
 325         char            *action_str[HIST_ACTIONS_MAX];
 326         unsigned int    n_actions;
 327
 328         struct var_defs var_defs;
 329 };
 330
 331 struct field_var {
 332         struct hist_field       *var;
 333         struct hist_field       *val;
 334 };
 335
 336 struct field_var_hist {
 337         struct hist_trigger_data        *hist_data;
 338         char                            *cmd;
 339 };
 340
 341 struct hist_trigger_data {
 342         struct hist_field               *fields[HIST_FIELDS_MAX];
 343         unsigned int                    n_vals;
 344         unsigned int                    n_keys;
 345         unsigned int                    n_fields;
 346         unsigned int                    n_vars;
 347         unsigned int                    key_size;
 348         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 349         unsigned int                    n_sort_keys;
 350         struct trace_event_file         *event_file;
 351         struct hist_trigger_attrs       *attrs;
 352         struct tracing_map              *map;
 353         bool                            enable_timestamps;
 354         bool                            remove;
 355         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 356         unsigned int                    n_var_refs;
 357
 358         struct action_data              *actions[HIST_ACTIONS_MAX];
 359         unsigned int                    n_actions;
 360
 361         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 362         unsigned int                    n_field_vars;
 363         unsigned int                    n_field_var_str;
 364         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 365         unsigned int                    n_field_var_hists;
 366
 367         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 368         unsigned int                    n_save_vars;
 369         unsigned int                    n_save_var_str;
 370 };
 371
 372 static int synth_event_create(int argc, const char **argv);
 373 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 374 static int synth_event_release(struct dyn_event *ev);
 375 static bool synth_event_is_busy(struct dyn_event *ev);
 376 static bool synth_event_match(const char *system, const char *event,
 377                               struct dyn_event *ev);
 378
 379 static struct dyn_event_operations synth_event_ops = {
 380         .create = synth_event_create,
 381         .show = synth_event_show,
 382         .is_busy = synth_event_is_busy,
 383         .free = synth_event_release,
 384         .match = synth_event_match,
 385 };
 386
 387 struct synth_field {
 388         char *type;
 389         char *name;
 390         size_t size;
 391         bool is_signed;
 392         bool is_string;
 393 };
 394
 395 struct synth_event {
 396         struct dyn_event                        devent;
 397         int                                     ref;
 398         char                                    *name;
 399         struct synth_field                      **fields;
 400         unsigned int                            n_fields;
 401         unsigned int                            n_u64;
 402         struct trace_event_class                class;
 403         struct trace_event_call                 call;
 404         struct tracepoint                       *tp;
 405 };
 406
 407 static bool is_synth_event(struct dyn_event *ev)
 408 {
 409         return ev->ops == &synth_event_ops;
 410 }
 411
 412 static struct synth_event *to_synth_event(struct dyn_event *ev)
 413 {
 414         return container_of(ev, struct synth_event, devent);
 415 }
 416
 417 static bool synth_event_is_busy(struct dyn_event *ev)
 418 {
 419         struct synth_event *event = to_synth_event(ev);
 420
 421         return event->ref != 0;
 422 }
 423
 424 static bool synth_event_match(const char *system, const char *event,
 425                               struct dyn_event *ev)
 426 {
 427         struct synth_event *sev = to_synth_event(ev);
 428
 429         return strcmp(sev->name, event) == 0 &&
 430                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 431 }
 432
 433 struct action_data;
 434
 435 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 436                              struct tracing_map_elt *elt, void *rec,
 437                              struct ring_buffer_event *rbe, void *key,
 438                              struct action_data *data, u64 *var_ref_vals);
 439
 440 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 441
 442 enum handler_id {
 443         HANDLER_ONMATCH = 1,
 444         HANDLER_ONMAX,
 445         HANDLER_ONCHANGE,
 446 };
 447
 448 enum action_id {
 449         ACTION_SAVE = 1,
 450         ACTION_TRACE,
 451         ACTION_SNAPSHOT,
 452 };
 453
 454 struct action_data {
 455         enum handler_id         handler;
 456         enum action_id          action;
 457         char                    *action_name;
 458         action_fn_t             fn;
 459
 460         unsigned int            n_params;
 461         char                    *params[SYNTH_FIELDS_MAX];
 462
 463         /*
 464          * When a histogram trigger is hit, the values of any
 465          * references to variables, including variables being passed
 466          * as parameters to synthetic events, are collected into a
 467          * var_ref_vals array.  This var_ref_idx is the index of the
 468          * first param in the array to be passed to the synthetic
 469          * event invocation.
 470          */
 471         unsigned int            var_ref_idx;
 472         struct synth_event      *synth_event;
 473         bool                    use_trace_keyword;
 474         char                    *synth_event_name;
 475
 476         union {
 477                 struct {
 478                         char                    *event;
 479                         char                    *event_system;
 480                 } match_data;
 481
 482                 struct {
 483                         /*
 484                          * var_str contains the $-unstripped variable
 485                          * name referenced by var_ref, and used when
 486                          * printing the action.  Because var_ref
 487                          * creation is deferred to create_actions(),
 488                          * we need a per-action way to save it until
 489                          * then, thus var_str.
 490                          */
 491                         char                    *var_str;
 492
 493                         /*
 494                          * var_ref refers to the variable being
 495                          * tracked e.g onmax($var).
 496                          */
 497                         struct hist_field       *var_ref;
 498
 499                         /*
 500                          * track_var contains the 'invisible' tracking
 501                          * variable created to keep the current
 502                          * e.g. max value.
 503                          */
 504                         struct hist_field       *track_var;
 505
 506                         check_track_val_fn_t    check_val;
 507                         action_fn_t             save_data;
 508                 } track_data;
 509         };
 510 };
 511
 512 struct track_data {
 513         u64                             track_val;
 514         bool                            updated;
 515
 516         unsigned int                    key_len;
 517         void                            *key;
 518         struct tracing_map_elt          elt;
 519
 520         struct action_data              *action_data;
 521         struct hist_trigger_data        *hist_data;
 522 };
 523
 524 struct hist_elt_data {
 525         char *comm;
 526         u64 *var_ref_vals;
 527         char *field_var_str[SYNTH_FIELDS_MAX];
 528 };
 529
 530 struct snapshot_context {
 531         struct tracing_map_elt  *elt;
 532         void                    *key;
 533 };
 534
 535 static void track_data_free(struct track_data *track_data)
 536 {
 537         struct hist_elt_data *elt_data;
 538
 539         if (!track_data)
 540                 return;
 541
 542         kfree(track_data->key);
 543
 544         elt_data = track_data->elt.private_data;
 545         if (elt_data) {
 546                 kfree(elt_data->comm);
 547                 kfree(elt_data);
 548         }
 549
 550         kfree(track_data);
 551 }
 552
 553 static struct track_data *track_data_alloc(unsigned int key_len,
 554                                            struct action_data *action_data,
 555                                            struct hist_trigger_data *hist_data)
 556 {
 557         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 558         struct hist_elt_data *elt_data;
 559
 560         if (!data)
 561                 return ERR_PTR(-ENOMEM);
 562
 563         data->key = kzalloc(key_len, GFP_KERNEL);
 564         if (!data->key) {
 565                 track_data_free(data);
 566                 return ERR_PTR(-ENOMEM);
 567         }
 568
 569         data->key_len = key_len;
 570         data->action_data = action_data;
 571         data->hist_data = hist_data;
 572
 573         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 574         if (!elt_data) {
 575                 track_data_free(data);
 576                 return ERR_PTR(-ENOMEM);
 577         }
 578         data->elt.private_data = elt_data;
 579
 580         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 581         if (!elt_data->comm) {
 582                 track_data_free(data);
 583                 return ERR_PTR(-ENOMEM);
 584         }
 585
 586         return data;
 587 }
 588
 589 static char last_cmd[MAX_FILTER_STR_VAL];
 590 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 591
 592 static int errpos(char *str)
 593 {
 594         return err_pos(last_cmd, str);
 595 }
 596
 597 static void last_cmd_set(struct trace_event_file *file, char *str)
 598 {
 599         const char *system = NULL, *name = NULL;
 600         struct trace_event_call *call;
 601
 602         if (!str)
 603                 return;
 604
 605         strncpy(last_cmd, str, MAX_FILTER_STR_VAL - 1);
 606
 607         if (file) {
 608                 call = file->event_call;
 609
 610                 system = call->class->system;
 611                 if (system) {
 612                         name = trace_event_name(call);
 613                         if (!name)
 614                                 system = NULL;
 615                 }
 616         }
 617
 618         if (system)
 619                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 620 }
 621
 622 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 623 {
 624         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 625                         err_type, err_pos);
 626 }
 627
 628 static void hist_err_clear(void)
 629 {
 630         last_cmd[0] = '\0';
 631         last_cmd_loc[0] = '\0';
 632 }
 633
 634 struct synth_trace_event {
 635         struct trace_entry      ent;
 636         u64                     fields[];
 637 };
 638
 639 static int synth_event_define_fields(struct trace_event_call *call)
 640 {
 641         struct synth_trace_event trace;
 642         int offset = offsetof(typeof(trace), fields);
 643         struct synth_event *event = call->data;
 644         unsigned int i, size, n_u64;
 645         char *name, *type;
 646         bool is_signed;
 647         int ret = 0;
 648
 649         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 650                 size = event->fields[i]->size;
 651                 is_signed = event->fields[i]->is_signed;
 652                 type = event->fields[i]->type;
 653                 name = event->fields[i]->name;
 654                 ret = trace_define_field(call, type, name, offset, size,
 655                                          is_signed, FILTER_OTHER);
 656                 if (ret)
 657                         break;
 658
 659                 if (event->fields[i]->is_string) {
 660                         offset += STR_VAR_LEN_MAX;
 661                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 662                 } else {
 663                         offset += sizeof(u64);
 664                         n_u64++;
 665                 }
 666         }
 667
 668         event->n_u64 = n_u64;
 669
 670         return ret;
 671 }
 672
 673 static bool synth_field_signed(char *type)
 674 {
 675         if (str_has_prefix(type, "u"))
 676                 return false;
 677
 678         return true;
 679 }
 680
 681 static int synth_field_is_string(char *type)
 682 {
 683         if (strstr(type, "char[") != NULL)
 684                 return true;
 685
 686         return false;
 687 }
 688
 689 static int synth_field_string_size(char *type)
 690 {
 691         char buf[4], *end, *start;
 692         unsigned int len;
 693         int size, err;
 694
 695         start = strstr(type, "char[");
 696         if (start == NULL)
 697                 return -EINVAL;
 698         start += sizeof("char[") - 1;
 699
 700         end = strchr(type, ']');
 701         if (!end || end < start)
 702                 return -EINVAL;
 703
 704         len = end - start;
 705         if (len > 3)
 706                 return -EINVAL;
 707
 708         strncpy(buf, start, len);
 709         buf[len] = '\0';
 710
 711         err = kstrtouint(buf, 0, &size);
 712         if (err)
 713                 return err;
 714
 715         if (size > STR_VAR_LEN_MAX)
 716                 return -EINVAL;
 717
 718         return size;
 719 }
 720
 721 static int synth_field_size(char *type)
 722 {
 723         int size = 0;
 724
 725         if (strcmp(type, "s64") == 0)
 726                 size = sizeof(s64);
 727         else if (strcmp(type, "u64") == 0)
 728                 size = sizeof(u64);
 729         else if (strcmp(type, "s32") == 0)
 730                 size = sizeof(s32);
 731         else if (strcmp(type, "u32") == 0)
 732                 size = sizeof(u32);
 733         else if (strcmp(type, "s16") == 0)
 734                 size = sizeof(s16);
 735         else if (strcmp(type, "u16") == 0)
 736                 size = sizeof(u16);
 737         else if (strcmp(type, "s8") == 0)
 738                 size = sizeof(s8);
 739         else if (strcmp(type, "u8") == 0)
 740                 size = sizeof(u8);
 741         else if (strcmp(type, "char") == 0)
 742                 size = sizeof(char);
 743         else if (strcmp(type, "unsigned char") == 0)
 744                 size = sizeof(unsigned char);
 745         else if (strcmp(type, "int") == 0)
 746                 size = sizeof(int);
 747         else if (strcmp(type, "unsigned int") == 0)
 748                 size = sizeof(unsigned int);
 749         else if (strcmp(type, "long") == 0)
 750                 size = sizeof(long);
 751         else if (strcmp(type, "unsigned long") == 0)
 752                 size = sizeof(unsigned long);
 753         else if (strcmp(type, "pid_t") == 0)
 754                 size = sizeof(pid_t);
 755         else if (synth_field_is_string(type))
 756                 size = synth_field_string_size(type);
 757
 758         return size;
 759 }
 760
 761 static const char *synth_field_fmt(char *type)
 762 {
 763         const char *fmt = "%llu";
 764
 765         if (strcmp(type, "s64") == 0)
 766                 fmt = "%lld";
 767         else if (strcmp(type, "u64") == 0)
 768                 fmt = "%llu";
 769         else if (strcmp(type, "s32") == 0)
 770                 fmt = "%d";
 771         else if (strcmp(type, "u32") == 0)
 772                 fmt = "%u";
 773         else if (strcmp(type, "s16") == 0)
 774                 fmt = "%d";
 775         else if (strcmp(type, "u16") == 0)
 776                 fmt = "%u";
 777         else if (strcmp(type, "s8") == 0)
 778                 fmt = "%d";
 779         else if (strcmp(type, "u8") == 0)
 780                 fmt = "%u";
 781         else if (strcmp(type, "char") == 0)
 782                 fmt = "%d";
 783         else if (strcmp(type, "unsigned char") == 0)
 784                 fmt = "%u";
 785         else if (strcmp(type, "int") == 0)
 786                 fmt = "%d";
 787         else if (strcmp(type, "unsigned int") == 0)
 788                 fmt = "%u";
 789         else if (strcmp(type, "long") == 0)
 790                 fmt = "%ld";
 791         else if (strcmp(type, "unsigned long") == 0)
 792                 fmt = "%lu";
 793         else if (strcmp(type, "pid_t") == 0)
 794                 fmt = "%d";
 795         else if (synth_field_is_string(type))
 796                 fmt = "%s";
 797
 798         return fmt;
 799 }
 800
 801 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 802                                            int flags,
 803                                            struct trace_event *event)
 804 {
 805         struct trace_array *tr = iter->tr;
 806         struct trace_seq *s = &iter->seq;
 807         struct synth_trace_event *entry;
 808         struct synth_event *se;
 809         unsigned int i, n_u64;
 810         char print_fmt[32];
 811         const char *fmt;
 812
 813         entry = (struct synth_trace_event *)iter->ent;
 814         se = container_of(event, struct synth_event, call.event);
 815
 816         trace_seq_printf(s, "%s: ", se->name);
 817
 818         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 819                 if (trace_seq_has_overflowed(s))
 820                         goto end;
 821
 822                 fmt = synth_field_fmt(se->fields[i]->type);
 823
 824                 /* parameter types */
 825                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
 826                         trace_seq_printf(s, "%s ", fmt);
 827
 828                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 829
 830                 /* parameter values */
 831                 if (se->fields[i]->is_string) {
 832                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 833                                          (char *)&entry->fields[n_u64],
 834                                          i == se->n_fields - 1 ? "" : " ");
 835                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 836                 } else {
 837                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 838                                          entry->fields[n_u64],
 839                                          i == se->n_fields - 1 ? "" : " ");
 840                         n_u64++;
 841                 }
 842         }
 843 end:
 844         trace_seq_putc(s, '\n');
 845
 846         return trace_handle_return(s);
 847 }
 848
 849 static struct trace_event_functions synth_event_funcs = {
 850         .trace          = print_synth_event
 851 };
 852
 853 static notrace void trace_event_raw_event_synth(void *__data,
 854                                                 u64 *var_ref_vals,
 855                                                 unsigned int var_ref_idx)
 856 {
 857         struct trace_event_file *trace_file = __data;
 858         struct synth_trace_event *entry;
 859         struct trace_event_buffer fbuffer;
 860         struct ring_buffer *buffer;
 861         struct synth_event *event;
 862         unsigned int i, n_u64;
 863         int fields_size = 0;
 864
 865         event = trace_file->event_call->data;
 866
 867         if (trace_trigger_soft_disabled(trace_file))
 868                 return;
 869
 870         fields_size = event->n_u64 * sizeof(u64);
 871
 872         /*
 873          * Avoid ring buffer recursion detection, as this event
 874          * is being performed within another event.
 875          */
 876         buffer = trace_file->tr->trace_buffer.buffer;
 877         ring_buffer_nest_start(buffer);
 878
 879         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 880                                            sizeof(*entry) + fields_size);
 881         if (!entry)
 882                 goto out;
 883
 884         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 885                 if (event->fields[i]->is_string) {
 886                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 887                         char *str_field = (char *)&entry->fields[n_u64];
 888
 889                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 890                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 891                 } else {
 892                         entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
 893                         n_u64++;
 894                 }
 895         }
 896
 897         trace_event_buffer_commit(&fbuffer);
 898 out:
 899         ring_buffer_nest_end(buffer);
 900 }
 901
 902 static void free_synth_event_print_fmt(struct trace_event_call *call)
 903 {
 904         if (call) {
 905                 kfree(call->print_fmt);
 906                 call->print_fmt = NULL;
 907         }
 908 }
 909
 910 static int __set_synth_event_print_fmt(struct synth_event *event,
 911                                        char *buf, int len)
 912 {
 913         const char *fmt;
 914         int pos = 0;
 915         int i;
 916
 917         /* When len=0, we just calculate the needed length */
 918 #define LEN_OR_ZERO (len ? len - pos : 0)
 919
 920         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 921         for (i = 0; i < event->n_fields; i++) {
 922                 fmt = synth_field_fmt(event->fields[i]->type);
 923                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 924                                 event->fields[i]->name, fmt,
 925                                 i == event->n_fields - 1 ? "" : ", ");
 926         }
 927         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 928
 929         for (i = 0; i < event->n_fields; i++) {
 930                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 931                                 ", REC->%s", event->fields[i]->name);
 932         }
 933
 934 #undef LEN_OR_ZERO
 935
 936         /* return the length of print_fmt */
 937         return pos;
 938 }
 939
 940 static int set_synth_event_print_fmt(struct trace_event_call *call)
 941 {
 942         struct synth_event *event = call->data;
 943         char *print_fmt;
 944         int len;
 945
 946         /* First: called with 0 length to calculate the needed length */
 947         len = __set_synth_event_print_fmt(event, NULL, 0);
 948
 949         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 950         if (!print_fmt)
 951                 return -ENOMEM;
 952
 953         /* Second: actually write the @print_fmt */
 954         __set_synth_event_print_fmt(event, print_fmt, len + 1);
 955         call->print_fmt = print_fmt;
 956
 957         return 0;
 958 }
 959
 960 static void free_synth_field(struct synth_field *field)
 961 {
 962         kfree(field->type);
 963         kfree(field->name);
 964         kfree(field);
 965 }
 966
 967 static struct synth_field *parse_synth_field(int argc, const char **argv,
 968                                              int *consumed)
 969 {
 970         struct synth_field *field;
 971         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
 972         int len, ret = 0;
 973
 974         if (field_type[0] == ';')
 975                 field_type++;
 976
 977         if (!strcmp(field_type, "unsigned")) {
 978                 if (argc < 3)
 979                         return ERR_PTR(-EINVAL);
 980                 prefix = "unsigned ";
 981                 field_type = argv[1];
 982                 field_name = argv[2];
 983                 *consumed = 3;
 984         } else {
 985                 field_name = argv[1];
 986                 *consumed = 2;
 987         }
 988
 989         field = kzalloc(sizeof(*field), GFP_KERNEL);
 990         if (!field)
 991                 return ERR_PTR(-ENOMEM);
 992
 993         len = strlen(field_name);
 994         array = strchr(field_name, '[');
 995         if (array)
 996                 len -= strlen(array);
 997         else if (field_name[len - 1] == ';')
 998                 len--;
 999
1000         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
1001         if (!field->name) {
1002                 ret = -ENOMEM;
1003                 goto free;
1004         }
1005
1006         if (field_type[0] == ';')
1007                 field_type++;
1008         len = strlen(field_type) + 1;
1009         if (array)
1010                 len += strlen(array);
1011         if (prefix)
1012                 len += strlen(prefix);
1013
1014         field->type = kzalloc(len, GFP_KERNEL);
1015         if (!field->type) {
1016                 ret = -ENOMEM;
1017                 goto free;
1018         }
1019         if (prefix)
1020                 strcat(field->type, prefix);
1021         strcat(field->type, field_type);
1022         if (array) {
1023                 strcat(field->type, array);
1024                 if (field->type[len - 1] == ';')
1025                         field->type[len - 1] = '\0';
1026         }
1027
1028         field->size = synth_field_size(field->type);
1029         if (!field->size) {
1030                 ret = -EINVAL;
1031                 goto free;
1032         }
1033
1034         if (synth_field_is_string(field->type))
1035                 field->is_string = true;
1036
1037         field->is_signed = synth_field_signed(field->type);
1038
1039  out:
1040         return field;
1041  free:
1042         free_synth_field(field);
1043         field = ERR_PTR(ret);
1044         goto out;
1045 }
1046
1047 static void free_synth_tracepoint(struct tracepoint *tp)
1048 {
1049         if (!tp)
1050                 return;
1051
1052         kfree(tp->name);
1053         kfree(tp);
1054 }
1055
1056 static struct tracepoint *alloc_synth_tracepoint(char *name)
1057 {
1058         struct tracepoint *tp;
1059
1060         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1061         if (!tp)
1062                 return ERR_PTR(-ENOMEM);
1063
1064         tp->name = kstrdup(name, GFP_KERNEL);
1065         if (!tp->name) {
1066                 kfree(tp);
1067                 return ERR_PTR(-ENOMEM);
1068         }
1069
1070         return tp;
1071 }
1072
1073 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1074                                     unsigned int var_ref_idx);
1075
1076 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1077                                unsigned int var_ref_idx)
1078 {
1079         struct tracepoint *tp = event->tp;
1080
1081         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1082                 struct tracepoint_func *probe_func_ptr;
1083                 synth_probe_func_t probe_func;
1084                 void *__data;
1085
1086                 if (!(cpu_online(raw_smp_processor_id())))
1087                         return;
1088
1089                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1090                 if (probe_func_ptr) {
1091                         do {
1092                                 probe_func = probe_func_ptr->func;
1093                                 __data = probe_func_ptr->data;
1094                                 probe_func(__data, var_ref_vals, var_ref_idx);
1095                         } while ((++probe_func_ptr)->func);
1096                 }
1097         }
1098 }
1099
1100 static struct synth_event *find_synth_event(const char *name)
1101 {
1102         struct dyn_event *pos;
1103         struct synth_event *event;
1104
1105         for_each_dyn_event(pos) {
1106                 if (!is_synth_event(pos))
1107                         continue;
1108                 event = to_synth_event(pos);
1109                 if (strcmp(event->name, name) == 0)
1110                         return event;
1111         }
1112
1113         return NULL;
1114 }
1115
1116 static int register_synth_event(struct synth_event *event)
1117 {
1118         struct trace_event_call *call = &event->call;
1119         int ret = 0;
1120
1121         event->call.class = &event->class;
1122         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1123         if (!event->class.system) {
1124                 ret = -ENOMEM;
1125                 goto out;
1126         }
1127
1128         event->tp = alloc_synth_tracepoint(event->name);
1129         if (IS_ERR(event->tp)) {
1130                 ret = PTR_ERR(event->tp);
1131                 event->tp = NULL;
1132                 goto out;
1133         }
1134
1135         INIT_LIST_HEAD(&call->class->fields);
1136         call->event.funcs = &synth_event_funcs;
1137         call->class->define_fields = synth_event_define_fields;
1138
1139         ret = register_trace_event(&call->event);
1140         if (!ret) {
1141                 ret = -ENODEV;
1142                 goto out;
1143         }
1144         call->flags = TRACE_EVENT_FL_TRACEPOINT;
1145         call->class->reg = trace_event_reg;
1146         call->class->probe = trace_event_raw_event_synth;
1147         call->data = event;
1148         call->tp = event->tp;
1149
1150         ret = trace_add_event_call(call);
1151         if (ret) {
1152                 pr_warn("Failed to register synthetic event: %s\n",
1153                         trace_event_name(call));
1154                 goto err;
1155         }
1156
1157         ret = set_synth_event_print_fmt(call);
1158         if (ret < 0) {
1159                 trace_remove_event_call(call);
1160                 goto err;
1161         }
1162  out:
1163         return ret;
1164  err:
1165         unregister_trace_event(&call->event);
1166         goto out;
1167 }
1168
1169 static int unregister_synth_event(struct synth_event *event)
1170 {
1171         struct trace_event_call *call = &event->call;
1172         int ret;
1173
1174         ret = trace_remove_event_call(call);
1175
1176         return ret;
1177 }
1178
1179 static void free_synth_event(struct synth_event *event)
1180 {
1181         unsigned int i;
1182
1183         if (!event)
1184                 return;
1185
1186         for (i = 0; i < event->n_fields; i++)
1187                 free_synth_field(event->fields[i]);
1188
1189         kfree(event->fields);
1190         kfree(event->name);
1191         kfree(event->class.system);
1192         free_synth_tracepoint(event->tp);
1193         free_synth_event_print_fmt(&event->call);
1194         kfree(event);
1195 }
1196
1197 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1198                                              struct synth_field **fields)
1199 {
1200         struct synth_event *event;
1201         unsigned int i;
1202
1203         event = kzalloc(sizeof(*event), GFP_KERNEL);
1204         if (!event) {
1205                 event = ERR_PTR(-ENOMEM);
1206                 goto out;
1207         }
1208
1209         event->name = kstrdup(name, GFP_KERNEL);
1210         if (!event->name) {
1211                 kfree(event);
1212                 event = ERR_PTR(-ENOMEM);
1213                 goto out;
1214         }
1215
1216         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1217         if (!event->fields) {
1218                 free_synth_event(event);
1219                 event = ERR_PTR(-ENOMEM);
1220                 goto out;
1221         }
1222
1223         dyn_event_init(&event->devent, &synth_event_ops);
1224
1225         for (i = 0; i < n_fields; i++)
1226                 event->fields[i] = fields[i];
1227
1228         event->n_fields = n_fields;
1229  out:
1230         return event;
1231 }
1232
1233 static void action_trace(struct hist_trigger_data *hist_data,
1234                          struct tracing_map_elt *elt, void *rec,
1235                          struct ring_buffer_event *rbe, void *key,
1236                          struct action_data *data, u64 *var_ref_vals)
1237 {
1238         struct synth_event *event = data->synth_event;
1239
1240         trace_synth(event, var_ref_vals, data->var_ref_idx);
1241 }
1242
1243 struct hist_var_data {
1244         struct list_head list;
1245         struct hist_trigger_data *hist_data;
1246 };
1247
1248 static int __create_synth_event(int argc, const char *name, const char **argv)
1249 {
1250         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1251         struct synth_event *event = NULL;
1252         int i, consumed = 0, n_fields = 0, ret = 0;
1253
1254         /*
1255          * Argument syntax:
1256          *  - Add synthetic event: <event_name> field[;field] ...
1257          *  - Remove synthetic event: !<event_name> field[;field] ...
1258          *      where 'field' = type field_name
1259          */
1260
1261         if (name[0] == '\0' || argc < 1)
1262                 return -EINVAL;
1263
1264         mutex_lock(&event_mutex);
1265
1266         event = find_synth_event(name);
1267         if (event) {
1268                 ret = -EEXIST;
1269                 goto out;
1270         }
1271
1272         for (i = 0; i < argc - 1; i++) {
1273                 if (strcmp(argv[i], ";") == 0)
1274                         continue;
1275                 if (n_fields == SYNTH_FIELDS_MAX) {
1276                         ret = -EINVAL;
1277                         goto err;
1278                 }
1279
1280                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1281                 if (IS_ERR(field)) {
1282                         ret = PTR_ERR(field);
1283                         goto err;
1284                 }
1285                 fields[n_fields++] = field;
1286                 i += consumed - 1;
1287         }
1288
1289         if (i < argc && strcmp(argv[i], ";") != 0) {
1290                 ret = -EINVAL;
1291                 goto err;
1292         }
1293
1294         event = alloc_synth_event(name, n_fields, fields);
1295         if (IS_ERR(event)) {
1296                 ret = PTR_ERR(event);
1297                 event = NULL;
1298                 goto err;
1299         }
1300         ret = register_synth_event(event);
1301         if (!ret)
1302                 dyn_event_add(&event->devent);
1303         else
1304                 free_synth_event(event);
1305  out:
1306         mutex_unlock(&event_mutex);
1307
1308         return ret;
1309  err:
1310         for (i = 0; i < n_fields; i++)
1311                 free_synth_field(fields[i]);
1312
1313         goto out;
1314 }
1315
1316 static int create_or_delete_synth_event(int argc, char **argv)
1317 {
1318         const char *name = argv[0];
1319         struct synth_event *event = NULL;
1320         int ret;
1321
1322         /* trace_run_command() ensures argc != 0 */
1323         if (name[0] == '!') {
1324                 mutex_lock(&event_mutex);
1325                 event = find_synth_event(name + 1);
1326                 if (event) {
1327                         if (event->ref)
1328                                 ret = -EBUSY;
1329                         else {
1330                                 ret = unregister_synth_event(event);
1331                                 if (!ret) {
1332                                         dyn_event_remove(&event->devent);
1333                                         free_synth_event(event);
1334                                 }
1335                         }
1336                 } else
1337                         ret = -ENOENT;
1338                 mutex_unlock(&event_mutex);
1339                 return ret;
1340         }
1341
1342         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1343         return ret == -ECANCELED ? -EINVAL : ret;
1344 }
1345
1346 static int synth_event_create(int argc, const char **argv)
1347 {
1348         const char *name = argv[0];
1349         int len;
1350
1351         if (name[0] != 's' || name[1] != ':')
1352                 return -ECANCELED;
1353         name += 2;
1354
1355         /* This interface accepts group name prefix */
1356         if (strchr(name, '/')) {
1357                 len = str_has_prefix(name, SYNTH_SYSTEM "/");
1358                 if (len == 0)
1359                         return -EINVAL;
1360                 name += len;
1361         }
1362         return __create_synth_event(argc - 1, name, argv + 1);
1363 }
1364
1365 static int synth_event_release(struct dyn_event *ev)
1366 {
1367         struct synth_event *event = to_synth_event(ev);
1368         int ret;
1369
1370         if (event->ref)
1371                 return -EBUSY;
1372
1373         ret = unregister_synth_event(event);
1374         if (ret)
1375                 return ret;
1376
1377         dyn_event_remove(ev);
1378         free_synth_event(event);
1379         return 0;
1380 }
1381
1382 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1383 {
1384         struct synth_field *field;
1385         unsigned int i;
1386
1387         seq_printf(m, "%s\t", event->name);
1388
1389         for (i = 0; i < event->n_fields; i++) {
1390                 field = event->fields[i];
1391
1392                 /* parameter values */
1393                 seq_printf(m, "%s %s%s", field->type, field->name,
1394                            i == event->n_fields - 1 ? "" : "; ");
1395         }
1396
1397         seq_putc(m, '\n');
1398
1399         return 0;
1400 }
1401
1402 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1403 {
1404         struct synth_event *event = to_synth_event(ev);
1405
1406         seq_printf(m, "s:%s/", event->class.system);
1407
1408         return __synth_event_show(m, event);
1409 }
1410
1411 static int synth_events_seq_show(struct seq_file *m, void *v)
1412 {
1413         struct dyn_event *ev = v;
1414
1415         if (!is_synth_event(ev))
1416                 return 0;
1417
1418         return __synth_event_show(m, to_synth_event(ev));
1419 }
1420
1421 static const struct seq_operations synth_events_seq_op = {
1422         .start  = dyn_event_seq_start,
1423         .next   = dyn_event_seq_next,
1424         .stop   = dyn_event_seq_stop,
1425         .show   = synth_events_seq_show,
1426 };
1427
1428 static int synth_events_open(struct inode *inode, struct file *file)
1429 {
1430         int ret;
1431
1432         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1433                 ret = dyn_events_release_all(&synth_event_ops);
1434                 if (ret < 0)
1435                         return ret;
1436         }
1437
1438         return seq_open(file, &synth_events_seq_op);
1439 }
1440
1441 static ssize_t synth_events_write(struct file *file,
1442                                   const char __user *buffer,
1443                                   size_t count, loff_t *ppos)
1444 {
1445         return trace_parse_run_command(file, buffer, count, ppos,
1446                                        create_or_delete_synth_event);
1447 }
1448
1449 static const struct file_operations synth_events_fops = {
1450         .open           = synth_events_open,
1451         .write          = synth_events_write,
1452         .read           = seq_read,
1453         .llseek         = seq_lseek,
1454         .release        = seq_release,
1455 };
1456
1457 static u64 hist_field_timestamp(struct hist_field *hist_field,
1458                                 struct tracing_map_elt *elt,
1459                                 struct ring_buffer_event *rbe,
1460                                 void *event)
1461 {
1462         struct hist_trigger_data *hist_data = hist_field->hist_data;
1463         struct trace_array *tr = hist_data->event_file->tr;
1464
1465         u64 ts = ring_buffer_event_time_stamp(rbe);
1466
1467         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1468                 ts = ns2usecs(ts);
1469
1470         return ts;
1471 }
1472
1473 static u64 hist_field_cpu(struct hist_field *hist_field,
1474                           struct tracing_map_elt *elt,
1475                           struct ring_buffer_event *rbe,
1476                           void *event)
1477 {
1478         int cpu = smp_processor_id();
1479
1480         return cpu;
1481 }
1482
1483 /**
1484  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1485  * @hist_field: The VAR_REF field to check
1486  * @var_data: The hist trigger that owns the variable
1487  * @var_idx: The trigger variable identifier
1488  *
1489  * Check the given VAR_REF field to see whether or not it references
1490  * the given variable associated with the given trigger.
1491  *
1492  * Return: The VAR_REF field if it does reference the variable, NULL if not
1493  */
1494 static struct hist_field *
1495 check_field_for_var_ref(struct hist_field *hist_field,
1496                         struct hist_trigger_data *var_data,
1497                         unsigned int var_idx)
1498 {
1499         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1500
1501         if (hist_field && hist_field->var.idx == var_idx &&
1502             hist_field->var.hist_data == var_data)
1503                 return hist_field;
1504
1505         return NULL;
1506 }
1507
1508 /**
1509  * find_var_ref - Check if a trigger has a reference to a trigger variable
1510  * @hist_data: The hist trigger that might have a reference to the variable
1511  * @var_data: The hist trigger that owns the variable
1512  * @var_idx: The trigger variable identifier
1513  *
1514  * Check the list of var_refs[] on the first hist trigger to see
1515  * whether any of them are references to the variable on the second
1516  * trigger.
1517  *
1518  * Return: The VAR_REF field referencing the variable if so, NULL if not
1519  */
1520 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1521                                        struct hist_trigger_data *var_data,
1522                                        unsigned int var_idx)
1523 {
1524         struct hist_field *hist_field;
1525         unsigned int i;
1526
1527         for (i = 0; i < hist_data->n_var_refs; i++) {
1528                 hist_field = hist_data->var_refs[i];
1529                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
1530                         return hist_field;
1531         }
1532
1533         return NULL;
1534 }
1535
1536 /**
1537  * find_any_var_ref - Check if there is a reference to a given trigger variable
1538  * @hist_data: The hist trigger
1539  * @var_idx: The trigger variable identifier
1540  *
1541  * Check to see whether the given variable is currently referenced by
1542  * any other trigger.
1543  *
1544  * The trigger the variable is defined on is explicitly excluded - the
1545  * assumption being that a self-reference doesn't prevent a trigger
1546  * from being removed.
1547  *
1548  * Return: The VAR_REF field referencing the variable if so, NULL if not
1549  */
1550 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1551                                            unsigned int var_idx)
1552 {
1553         struct trace_array *tr = hist_data->event_file->tr;
1554         struct hist_field *found = NULL;
1555         struct hist_var_data *var_data;
1556
1557         list_for_each_entry(var_data, &tr->hist_vars, list) {
1558                 if (var_data->hist_data == hist_data)
1559                         continue;
1560                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1561                 if (found)
1562                         break;
1563         }
1564
1565         return found;
1566 }
1567
1568 /**
1569  * check_var_refs - Check if there is a reference to any of trigger's variables
1570  * @hist_data: The hist trigger
1571  *
1572  * A trigger can define one or more variables.  If any one of them is
1573  * currently referenced by any other trigger, this function will
1574  * determine that.
1575
1576  * Typically used to determine whether or not a trigger can be removed
1577  * - if there are any references to a trigger's variables, it cannot.
1578  *
1579  * Return: True if there is a reference to any of trigger's variables
1580  */
1581 static bool check_var_refs(struct hist_trigger_data *hist_data)
1582 {
1583         struct hist_field *field;
1584         bool found = false;
1585         int i;
1586
1587         for_each_hist_field(i, hist_data) {
1588                 field = hist_data->fields[i];
1589                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1590                         if (find_any_var_ref(hist_data, field->var.idx)) {
1591                                 found = true;
1592                                 break;
1593                         }
1594                 }
1595         }
1596
1597         return found;
1598 }
1599
1600 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1601 {
1602         struct trace_array *tr = hist_data->event_file->tr;
1603         struct hist_var_data *var_data, *found = NULL;
1604
1605         list_for_each_entry(var_data, &tr->hist_vars, list) {
1606                 if (var_data->hist_data == hist_data) {
1607                         found = var_data;
1608                         break;
1609                 }
1610         }
1611
1612         return found;
1613 }
1614
1615 static bool field_has_hist_vars(struct hist_field *hist_field,
1616                                 unsigned int level)
1617 {
1618         int i;
1619
1620         if (level > 3)
1621                 return false;
1622
1623         if (!hist_field)
1624                 return false;
1625
1626         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1627             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1628                 return true;
1629
1630         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1631                 struct hist_field *operand;
1632
1633                 operand = hist_field->operands[i];
1634                 if (field_has_hist_vars(operand, level + 1))
1635                         return true;
1636         }
1637
1638         return false;
1639 }
1640
1641 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1642 {
1643         struct hist_field *hist_field;
1644         int i;
1645
1646         for_each_hist_field(i, hist_data) {
1647                 hist_field = hist_data->fields[i];
1648                 if (field_has_hist_vars(hist_field, 0))
1649                         return true;
1650         }
1651
1652         return false;
1653 }
1654
1655 static int save_hist_vars(struct hist_trigger_data *hist_data)
1656 {
1657         struct trace_array *tr = hist_data->event_file->tr;
1658         struct hist_var_data *var_data;
1659
1660         var_data = find_hist_vars(hist_data);
1661         if (var_data)
1662                 return 0;
1663
1664         if (trace_array_get(tr) < 0)
1665                 return -ENODEV;
1666
1667         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1668         if (!var_data) {
1669                 trace_array_put(tr);
1670                 return -ENOMEM;
1671         }
1672
1673         var_data->hist_data = hist_data;
1674         list_add(&var_data->list, &tr->hist_vars);
1675
1676         return 0;
1677 }
1678
1679 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1680 {
1681         struct trace_array *tr = hist_data->event_file->tr;
1682         struct hist_var_data *var_data;
1683
1684         var_data = find_hist_vars(hist_data);
1685         if (!var_data)
1686                 return;
1687
1688         if (WARN_ON(check_var_refs(hist_data)))
1689                 return;
1690
1691         list_del(&var_data->list);
1692
1693         kfree(var_data);
1694
1695         trace_array_put(tr);
1696 }
1697
1698 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1699                                          const char *var_name)
1700 {
1701         struct hist_field *hist_field, *found = NULL;
1702         int i;
1703
1704         for_each_hist_field(i, hist_data) {
1705                 hist_field = hist_data->fields[i];
1706                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1707                     strcmp(hist_field->var.name, var_name) == 0) {
1708                         found = hist_field;
1709                         break;
1710                 }
1711         }
1712
1713         return found;
1714 }
1715
1716 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1717                                    struct trace_event_file *file,
1718                                    const char *var_name)
1719 {
1720         struct hist_trigger_data *test_data;
1721         struct event_trigger_data *test;
1722         struct hist_field *hist_field;
1723
1724         hist_field = find_var_field(hist_data, var_name);
1725         if (hist_field)
1726                 return hist_field;
1727
1728         list_for_each_entry_rcu(test, &file->triggers, list) {
1729                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1730                         test_data = test->private_data;
1731                         hist_field = find_var_field(test_data, var_name);
1732                         if (hist_field)
1733                                 return hist_field;
1734                 }
1735         }
1736
1737         return NULL;
1738 }
1739
1740 static struct trace_event_file *find_var_file(struct trace_array *tr,
1741                                               char *system,
1742                                               char *event_name,
1743                                               char *var_name)
1744 {
1745         struct hist_trigger_data *var_hist_data;
1746         struct hist_var_data *var_data;
1747         struct trace_event_file *file, *found = NULL;
1748
1749         if (system)
1750                 return find_event_file(tr, system, event_name);
1751
1752         list_for_each_entry(var_data, &tr->hist_vars, list) {
1753                 var_hist_data = var_data->hist_data;
1754                 file = var_hist_data->event_file;
1755                 if (file == found)
1756                         continue;
1757
1758                 if (find_var_field(var_hist_data, var_name)) {
1759                         if (found) {
1760                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1761                                 return NULL;
1762                         }
1763
1764                         found = file;
1765                 }
1766         }
1767
1768         return found;
1769 }
1770
1771 static struct hist_field *find_file_var(struct trace_event_file *file,
1772                                         const char *var_name)
1773 {
1774         struct hist_trigger_data *test_data;
1775         struct event_trigger_data *test;
1776         struct hist_field *hist_field;
1777
1778         list_for_each_entry_rcu(test, &file->triggers, list) {
1779                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1780                         test_data = test->private_data;
1781                         hist_field = find_var_field(test_data, var_name);
1782                         if (hist_field)
1783                                 return hist_field;
1784                 }
1785         }
1786
1787         return NULL;
1788 }
1789
1790 static struct hist_field *
1791 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1792 {
1793         struct trace_array *tr = hist_data->event_file->tr;
1794         struct hist_field *hist_field, *found = NULL;
1795         struct trace_event_file *file;
1796         unsigned int i;
1797
1798         for (i = 0; i < hist_data->n_actions; i++) {
1799                 struct action_data *data = hist_data->actions[i];
1800
1801                 if (data->handler == HANDLER_ONMATCH) {
1802                         char *system = data->match_data.event_system;
1803                         char *event_name = data->match_data.event;
1804
1805                         file = find_var_file(tr, system, event_name, var_name);
1806                         if (!file)
1807                                 continue;
1808                         hist_field = find_file_var(file, var_name);
1809                         if (hist_field) {
1810                                 if (found) {
1811                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1812                                                  errpos(var_name));
1813                                         return ERR_PTR(-EINVAL);
1814                                 }
1815
1816                                 found = hist_field;
1817                         }
1818                 }
1819         }
1820         return found;
1821 }
1822
1823 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1824                                          char *system,
1825                                          char *event_name,
1826                                          char *var_name)
1827 {
1828         struct trace_array *tr = hist_data->event_file->tr;
1829         struct hist_field *hist_field = NULL;
1830         struct trace_event_file *file;
1831
1832         if (!system || !event_name) {
1833                 hist_field = find_match_var(hist_data, var_name);
1834                 if (IS_ERR(hist_field))
1835                         return NULL;
1836                 if (hist_field)
1837                         return hist_field;
1838         }
1839
1840         file = find_var_file(tr, system, event_name, var_name);
1841         if (!file)
1842                 return NULL;
1843
1844         hist_field = find_file_var(file, var_name);
1845
1846         return hist_field;
1847 }
1848
1849 static u64 hist_field_var_ref(struct hist_field *hist_field,
1850                               struct tracing_map_elt *elt,
1851                               struct ring_buffer_event *rbe,
1852                               void *event)
1853 {
1854         struct hist_elt_data *elt_data;
1855         u64 var_val = 0;
1856
1857         elt_data = elt->private_data;
1858         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1859
1860         return var_val;
1861 }
1862
1863 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1864                              u64 *var_ref_vals, bool self)
1865 {
1866         struct hist_trigger_data *var_data;
1867         struct tracing_map_elt *var_elt;
1868         struct hist_field *hist_field;
1869         unsigned int i, var_idx;
1870         bool resolved = true;
1871         u64 var_val = 0;
1872
1873         for (i = 0; i < hist_data->n_var_refs; i++) {
1874                 hist_field = hist_data->var_refs[i];
1875                 var_idx = hist_field->var.idx;
1876                 var_data = hist_field->var.hist_data;
1877
1878                 if (var_data == NULL) {
1879                         resolved = false;
1880                         break;
1881                 }
1882
1883                 if ((self && var_data != hist_data) ||
1884                     (!self && var_data == hist_data))
1885                         continue;
1886
1887                 var_elt = tracing_map_lookup(var_data->map, key);
1888                 if (!var_elt) {
1889                         resolved = false;
1890                         break;
1891                 }
1892
1893                 if (!tracing_map_var_set(var_elt, var_idx)) {
1894                         resolved = false;
1895                         break;
1896                 }
1897
1898                 if (self || !hist_field->read_once)
1899                         var_val = tracing_map_read_var(var_elt, var_idx);
1900                 else
1901                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1902
1903                 var_ref_vals[i] = var_val;
1904         }
1905
1906         return resolved;
1907 }
1908
1909 static const char *hist_field_name(struct hist_field *field,
1910                                    unsigned int level)
1911 {
1912         const char *field_name = "";
1913
1914         if (level > 1)
1915                 return field_name;
1916
1917         if (field->field)
1918                 field_name = field->field->name;
1919         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1920                  field->flags & HIST_FIELD_FL_ALIAS)
1921                 field_name = hist_field_name(field->operands[0], ++level);
1922         else if (field->flags & HIST_FIELD_FL_CPU)
1923                 field_name = "cpu";
1924         else if (field->flags & HIST_FIELD_FL_EXPR ||
1925                  field->flags & HIST_FIELD_FL_VAR_REF) {
1926                 if (field->system) {
1927                         static char full_name[MAX_FILTER_STR_VAL];
1928
1929                         strcat(full_name, field->system);
1930                         strcat(full_name, ".");
1931                         strcat(full_name, field->event_name);
1932                         strcat(full_name, ".");
1933                         strcat(full_name, field->name);
1934                         field_name = full_name;
1935                 } else
1936                         field_name = field->name;
1937         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1938                 field_name = "common_timestamp";
1939
1940         if (field_name == NULL)
1941                 field_name = "";
1942
1943         return field_name;
1944 }
1945
1946 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1947 {
1948         hist_field_fn_t fn = NULL;
1949
1950         switch (field_size) {
1951         case 8:
1952                 if (field_is_signed)
1953                         fn = hist_field_s64;
1954                 else
1955                         fn = hist_field_u64;
1956                 break;
1957         case 4:
1958                 if (field_is_signed)
1959                         fn = hist_field_s32;
1960                 else
1961                         fn = hist_field_u32;
1962                 break;
1963         case 2:
1964                 if (field_is_signed)
1965                         fn = hist_field_s16;
1966                 else
1967                         fn = hist_field_u16;
1968                 break;
1969         case 1:
1970                 if (field_is_signed)
1971                         fn = hist_field_s8;
1972                 else
1973                         fn = hist_field_u8;
1974                 break;
1975         }
1976
1977         return fn;
1978 }
1979
1980 static int parse_map_size(char *str)
1981 {
1982         unsigned long size, map_bits;
1983         int ret;
1984
1985         strsep(&str, "=");
1986         if (!str) {
1987                 ret = -EINVAL;
1988                 goto out;
1989         }
1990
1991         ret = kstrtoul(str, 0, &size);
1992         if (ret)
1993                 goto out;
1994
1995         map_bits = ilog2(roundup_pow_of_two(size));
1996         if (map_bits < TRACING_MAP_BITS_MIN ||
1997             map_bits > TRACING_MAP_BITS_MAX)
1998                 ret = -EINVAL;
1999         else
2000                 ret = map_bits;
2001  out:
2002         return ret;
2003 }
2004
2005 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
2006 {
2007         unsigned int i;
2008
2009         if (!attrs)
2010                 return;
2011
2012         for (i = 0; i < attrs->n_assignments; i++)
2013                 kfree(attrs->assignment_str[i]);
2014
2015         for (i = 0; i < attrs->n_actions; i++)
2016                 kfree(attrs->action_str[i]);
2017
2018         kfree(attrs->name);
2019         kfree(attrs->sort_key_str);
2020         kfree(attrs->keys_str);
2021         kfree(attrs->vals_str);
2022         kfree(attrs->clock);
2023         kfree(attrs);
2024 }
2025
2026 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
2027 {
2028         int ret = -EINVAL;
2029
2030         if (attrs->n_actions >= HIST_ACTIONS_MAX)
2031                 return ret;
2032
2033         if ((str_has_prefix(str, "onmatch(")) ||
2034             (str_has_prefix(str, "onmax(")) ||
2035             (str_has_prefix(str, "onchange("))) {
2036                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
2037                 if (!attrs->action_str[attrs->n_actions]) {
2038                         ret = -ENOMEM;
2039                         return ret;
2040                 }
2041                 attrs->n_actions++;
2042                 ret = 0;
2043         }
2044         return ret;
2045 }
2046
2047 static int parse_assignment(struct trace_array *tr,
2048                             char *str, struct hist_trigger_attrs *attrs)
2049 {
2050         int ret = 0;
2051
2052         if ((str_has_prefix(str, "key=")) ||
2053             (str_has_prefix(str, "keys="))) {
2054                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
2055                 if (!attrs->keys_str) {
2056                         ret = -ENOMEM;
2057                         goto out;
2058                 }
2059         } else if ((str_has_prefix(str, "val=")) ||
2060                    (str_has_prefix(str, "vals=")) ||
2061                    (str_has_prefix(str, "values="))) {
2062                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
2063                 if (!attrs->vals_str) {
2064                         ret = -ENOMEM;
2065                         goto out;
2066                 }
2067         } else if (str_has_prefix(str, "sort=")) {
2068                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
2069                 if (!attrs->sort_key_str) {
2070                         ret = -ENOMEM;
2071                         goto out;
2072                 }
2073         } else if (str_has_prefix(str, "name=")) {
2074                 attrs->name = kstrdup(str, GFP_KERNEL);
2075                 if (!attrs->name) {
2076                         ret = -ENOMEM;
2077                         goto out;
2078                 }
2079         } else if (str_has_prefix(str, "clock=")) {
2080                 strsep(&str, "=");
2081                 if (!str) {
2082                         ret = -EINVAL;
2083                         goto out;
2084                 }
2085
2086                 str = strstrip(str);
2087                 attrs->clock = kstrdup(str, GFP_KERNEL);
2088                 if (!attrs->clock) {
2089                         ret = -ENOMEM;
2090                         goto out;
2091                 }
2092         } else if (str_has_prefix(str, "size=")) {
2093                 int map_bits = parse_map_size(str);
2094
2095                 if (map_bits < 0) {
2096                         ret = map_bits;
2097                         goto out;
2098                 }
2099                 attrs->map_bits = map_bits;
2100         } else {
2101                 char *assignment;
2102
2103                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
2104                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
2105                         ret = -EINVAL;
2106                         goto out;
2107                 }
2108
2109                 assignment = kstrdup(str, GFP_KERNEL);
2110                 if (!assignment) {
2111                         ret = -ENOMEM;
2112                         goto out;
2113                 }
2114
2115                 attrs->assignment_str[attrs->n_assignments++] = assignment;
2116         }
2117  out:
2118         return ret;
2119 }
2120
2121 static struct hist_trigger_attrs *
2122 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
2123 {
2124         struct hist_trigger_attrs *attrs;
2125         int ret = 0;
2126
2127         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
2128         if (!attrs)
2129                 return ERR_PTR(-ENOMEM);
2130
2131         while (trigger_str) {
2132                 char *str = strsep(&trigger_str, ":");
2133
2134                 if (strchr(str, '=')) {
2135                         ret = parse_assignment(tr, str, attrs);
2136                         if (ret)
2137                                 goto free;
2138                 } else if (strcmp(str, "pause") == 0)
2139                         attrs->pause = true;
2140                 else if ((strcmp(str, "cont") == 0) ||
2141                          (strcmp(str, "continue") == 0))
2142                         attrs->cont = true;
2143                 else if (strcmp(str, "clear") == 0)
2144                         attrs->clear = true;
2145                 else {
2146                         ret = parse_action(str, attrs);
2147                         if (ret)
2148                                 goto free;
2149                 }
2150         }
2151
2152         if (!attrs->keys_str) {
2153                 ret = -EINVAL;
2154                 goto free;
2155         }
2156
2157         if (!attrs->clock) {
2158                 attrs->clock = kstrdup("global", GFP_KERNEL);
2159                 if (!attrs->clock) {
2160                         ret = -ENOMEM;
2161                         goto free;
2162                 }
2163         }
2164
2165         return attrs;
2166  free:
2167         destroy_hist_trigger_attrs(attrs);
2168
2169         return ERR_PTR(ret);
2170 }
2171
2172 static inline void save_comm(char *comm, struct task_struct *task)
2173 {
2174         if (!task->pid) {
2175                 strcpy(comm, "<idle>");
2176                 return;
2177         }
2178
2179         if (WARN_ON_ONCE(task->pid < 0)) {
2180                 strcpy(comm, "<XXX>");
2181                 return;
2182         }
2183
2184         strncpy(comm, task->comm, TASK_COMM_LEN);
2185 }
2186
2187 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2188 {
2189         unsigned int i;
2190
2191         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2192                 kfree(elt_data->field_var_str[i]);
2193
2194         kfree(elt_data->comm);
2195         kfree(elt_data);
2196 }
2197
2198 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2199 {
2200         struct hist_elt_data *elt_data = elt->private_data;
2201
2202         hist_elt_data_free(elt_data);
2203 }
2204
2205 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2206 {
2207         struct hist_trigger_data *hist_data = elt->map->private_data;
2208         unsigned int size = TASK_COMM_LEN;
2209         struct hist_elt_data *elt_data;
2210         struct hist_field *key_field;
2211         unsigned int i, n_str;
2212
2213         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2214         if (!elt_data)
2215                 return -ENOMEM;
2216
2217         for_each_hist_key_field(i, hist_data) {
2218                 key_field = hist_data->fields[i];
2219
2220                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2221                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2222                         if (!elt_data->comm) {
2223                                 kfree(elt_data);
2224                                 return -ENOMEM;
2225                         }
2226                         break;
2227                 }
2228         }
2229
2230         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
2231
2232         size = STR_VAR_LEN_MAX;
2233
2234         for (i = 0; i < n_str; i++) {
2235                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2236                 if (!elt_data->field_var_str[i]) {
2237                         hist_elt_data_free(elt_data);
2238                         return -ENOMEM;
2239                 }
2240         }
2241
2242         elt->private_data = elt_data;
2243
2244         return 0;
2245 }
2246
2247 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2248 {
2249         struct hist_elt_data *elt_data = elt->private_data;
2250
2251         if (elt_data->comm)
2252                 save_comm(elt_data->comm, current);
2253 }
2254
2255 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2256         .elt_alloc      = hist_trigger_elt_data_alloc,
2257         .elt_free       = hist_trigger_elt_data_free,
2258         .elt_init       = hist_trigger_elt_data_init,
2259 };
2260
2261 static const char *get_hist_field_flags(struct hist_field *hist_field)
2262 {
2263         const char *flags_str = NULL;
2264
2265         if (hist_field->flags & HIST_FIELD_FL_HEX)
2266                 flags_str = "hex";
2267         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2268                 flags_str = "sym";
2269         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2270                 flags_str = "sym-offset";
2271         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2272                 flags_str = "execname";
2273         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2274                 flags_str = "syscall";
2275         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2276                 flags_str = "log2";
2277         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2278                 flags_str = "usecs";
2279
2280         return flags_str;
2281 }
2282
2283 static void expr_field_str(struct hist_field *field, char *expr)
2284 {
2285         if (field->flags & HIST_FIELD_FL_VAR_REF)
2286                 strcat(expr, "$");
2287
2288         strcat(expr, hist_field_name(field, 0));
2289
2290         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2291                 const char *flags_str = get_hist_field_flags(field);
2292
2293                 if (flags_str) {
2294                         strcat(expr, ".");
2295                         strcat(expr, flags_str);
2296                 }
2297         }
2298 }
2299
2300 static char *expr_str(struct hist_field *field, unsigned int level)
2301 {
2302         char *expr;
2303
2304         if (level > 1)
2305                 return NULL;
2306
2307         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2308         if (!expr)
2309                 return NULL;
2310
2311         if (!field->operands[0]) {
2312                 expr_field_str(field, expr);
2313                 return expr;
2314         }
2315
2316         if (field->operator == FIELD_OP_UNARY_MINUS) {
2317                 char *subexpr;
2318
2319                 strcat(expr, "-(");
2320                 subexpr = expr_str(field->operands[0], ++level);
2321                 if (!subexpr) {
2322                         kfree(expr);
2323                         return NULL;
2324                 }
2325                 strcat(expr, subexpr);
2326                 strcat(expr, ")");
2327
2328                 kfree(subexpr);
2329
2330                 return expr;
2331         }
2332
2333         expr_field_str(field->operands[0], expr);
2334
2335         switch (field->operator) {
2336         case FIELD_OP_MINUS:
2337                 strcat(expr, "-");
2338                 break;
2339         case FIELD_OP_PLUS:
2340                 strcat(expr, "+");
2341                 break;
2342         default:
2343                 kfree(expr);
2344                 return NULL;
2345         }
2346
2347         expr_field_str(field->operands[1], expr);
2348
2349         return expr;
2350 }
2351
2352 static int contains_operator(char *str)
2353 {
2354         enum field_op_id field_op = FIELD_OP_NONE;
2355         char *op;
2356
2357         op = strpbrk(str, "+-");
2358         if (!op)
2359                 return FIELD_OP_NONE;
2360
2361         switch (*op) {
2362         case '-':
2363                 if (*str == '-')
2364                         field_op = FIELD_OP_UNARY_MINUS;
2365                 else
2366                         field_op = FIELD_OP_MINUS;
2367                 break;
2368         case '+':
2369                 field_op = FIELD_OP_PLUS;
2370                 break;
2371         default:
2372                 break;
2373         }
2374
2375         return field_op;
2376 }
2377
2378 static void __destroy_hist_field(struct hist_field *hist_field)
2379 {
2380         kfree(hist_field->var.name);
2381         kfree(hist_field->name);
2382         kfree(hist_field->type);
2383
2384         kfree(hist_field);
2385 }
2386
2387 static void destroy_hist_field(struct hist_field *hist_field,
2388                                unsigned int level)
2389 {
2390         unsigned int i;
2391
2392         if (level > 3)
2393                 return;
2394
2395         if (!hist_field)
2396                 return;
2397
2398         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2399                 return; /* var refs will be destroyed separately */
2400
2401         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2402                 destroy_hist_field(hist_field->operands[i], level + 1);
2403
2404         __destroy_hist_field(hist_field);
2405 }
2406
2407 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2408                                             struct ftrace_event_field *field,
2409                                             unsigned long flags,
2410                                             char *var_name)
2411 {
2412         struct hist_field *hist_field;
2413
2414         if (field && is_function_field(field))
2415                 return NULL;
2416
2417         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2418         if (!hist_field)
2419                 return NULL;
2420
2421         hist_field->hist_data = hist_data;
2422
2423         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2424                 goto out; /* caller will populate */
2425
2426         if (flags & HIST_FIELD_FL_VAR_REF) {
2427                 hist_field->fn = hist_field_var_ref;
2428                 goto out;
2429         }
2430
2431         if (flags & HIST_FIELD_FL_HITCOUNT) {
2432                 hist_field->fn = hist_field_counter;
2433                 hist_field->size = sizeof(u64);
2434                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2435                 if (!hist_field->type)
2436                         goto free;
2437                 goto out;
2438         }
2439
2440         if (flags & HIST_FIELD_FL_STACKTRACE) {
2441                 hist_field->fn = hist_field_none;
2442                 goto out;
2443         }
2444
2445         if (flags & HIST_FIELD_FL_LOG2) {
2446                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2447                 hist_field->fn = hist_field_log2;
2448                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2449                 hist_field->size = hist_field->operands[0]->size;
2450                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2451                 if (!hist_field->type)
2452                         goto free;
2453                 goto out;
2454         }
2455
2456         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2457                 hist_field->fn = hist_field_timestamp;
2458                 hist_field->size = sizeof(u64);
2459                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2460                 if (!hist_field->type)
2461                         goto free;
2462                 goto out;
2463         }
2464
2465         if (flags & HIST_FIELD_FL_CPU) {
2466                 hist_field->fn = hist_field_cpu;
2467                 hist_field->size = sizeof(int);
2468                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2469                 if (!hist_field->type)
2470                         goto free;
2471                 goto out;
2472         }
2473
2474         if (WARN_ON_ONCE(!field))
2475                 goto out;
2476
2477         if (is_string_field(field)) {
2478                 flags |= HIST_FIELD_FL_STRING;
2479
2480                 hist_field->size = MAX_FILTER_STR_VAL;
2481                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2482                 if (!hist_field->type)
2483                         goto free;
2484
2485                 if (field->filter_type == FILTER_STATIC_STRING)
2486                         hist_field->fn = hist_field_string;
2487                 else if (field->filter_type == FILTER_DYN_STRING)
2488                         hist_field->fn = hist_field_dynstring;
2489                 else
2490                         hist_field->fn = hist_field_pstring;
2491         } else {
2492                 hist_field->size = field->size;
2493                 hist_field->is_signed = field->is_signed;
2494                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2495                 if (!hist_field->type)
2496                         goto free;
2497
2498                 hist_field->fn = select_value_fn(field->size,
2499                                                  field->is_signed);
2500                 if (!hist_field->fn) {
2501                         destroy_hist_field(hist_field, 0);
2502                         return NULL;
2503                 }
2504         }
2505  out:
2506         hist_field->field = field;
2507         hist_field->flags = flags;
2508
2509         if (var_name) {
2510                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2511                 if (!hist_field->var.name)
2512                         goto free;
2513         }
2514
2515         return hist_field;
2516  free:
2517         destroy_hist_field(hist_field, 0);
2518         return NULL;
2519 }
2520
2521 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2522 {
2523         unsigned int i;
2524
2525         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2526                 if (hist_data->fields[i]) {
2527                         destroy_hist_field(hist_data->fields[i], 0);
2528                         hist_data->fields[i] = NULL;
2529                 }
2530         }
2531
2532         for (i = 0; i < hist_data->n_var_refs; i++) {
2533                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2534                 __destroy_hist_field(hist_data->var_refs[i]);
2535                 hist_data->var_refs[i] = NULL;
2536         }
2537 }
2538
2539 static int init_var_ref(struct hist_field *ref_field,
2540                         struct hist_field *var_field,
2541                         char *system, char *event_name)
2542 {
2543         int err = 0;
2544
2545         ref_field->var.idx = var_field->var.idx;
2546         ref_field->var.hist_data = var_field->hist_data;
2547         ref_field->size = var_field->size;
2548         ref_field->is_signed = var_field->is_signed;
2549         ref_field->flags |= var_field->flags &
2550                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2551
2552         if (system) {
2553                 ref_field->system = kstrdup(system, GFP_KERNEL);
2554                 if (!ref_field->system)
2555                         return -ENOMEM;
2556         }
2557
2558         if (event_name) {
2559                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2560                 if (!ref_field->event_name) {
2561                         err = -ENOMEM;
2562                         goto free;
2563                 }
2564         }
2565
2566         if (var_field->var.name) {
2567                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2568                 if (!ref_field->name) {
2569                         err = -ENOMEM;
2570                         goto free;
2571                 }
2572         } else if (var_field->name) {
2573                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2574                 if (!ref_field->name) {
2575                         err = -ENOMEM;
2576                         goto free;
2577                 }
2578         }
2579
2580         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2581         if (!ref_field->type) {
2582                 err = -ENOMEM;
2583                 goto free;
2584         }
2585  out:
2586         return err;
2587  free:
2588         kfree(ref_field->system);
2589         kfree(ref_field->event_name);
2590         kfree(ref_field->name);
2591
2592         goto out;
2593 }
2594
2595 /**
2596  * create_var_ref - Create a variable reference and attach it to trigger
2597  * @hist_data: The trigger that will be referencing the variable
2598  * @var_field: The VAR field to create a reference to
2599  * @system: The optional system string
2600  * @event_name: The optional event_name string
2601  *
2602  * Given a variable hist_field, create a VAR_REF hist_field that
2603  * represents a reference to it.
2604  *
2605  * This function also adds the reference to the trigger that
2606  * now references the variable.
2607  *
2608  * Return: The VAR_REF field if successful, NULL if not
2609  */
2610 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2611                                          struct hist_field *var_field,
2612                                          char *system, char *event_name)
2613 {
2614         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2615         struct hist_field *ref_field;
2616
2617         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2618         if (ref_field) {
2619                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2620                         destroy_hist_field(ref_field, 0);
2621                         return NULL;
2622                 }
2623
2624                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2625                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2626         }
2627
2628         return ref_field;
2629 }
2630
2631 static bool is_var_ref(char *var_name)
2632 {
2633         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2634                 return false;
2635
2636         return true;
2637 }
2638
2639 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2640                                  char *var_name)
2641 {
2642         char *name, *field;
2643         unsigned int i;
2644
2645         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2646                 name = hist_data->attrs->var_defs.name[i];
2647
2648                 if (strcmp(var_name, name) == 0) {
2649                         field = hist_data->attrs->var_defs.expr[i];
2650                         if (contains_operator(field) || is_var_ref(field))
2651                                 continue;
2652                         return field;
2653                 }
2654         }
2655
2656         return NULL;
2657 }
2658
2659 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2660                                  char *system, char *event_name,
2661                                  char *var_name)
2662 {
2663         struct trace_event_call *call;
2664
2665         if (system && event_name) {
2666                 call = hist_data->event_file->event_call;
2667
2668                 if (strcmp(system, call->class->system) != 0)
2669                         return NULL;
2670
2671                 if (strcmp(event_name, trace_event_name(call)) != 0)
2672                         return NULL;
2673         }
2674
2675         if (!!system != !!event_name)
2676                 return NULL;
2677
2678         if (!is_var_ref(var_name))
2679                 return NULL;
2680
2681         var_name++;
2682
2683         return field_name_from_var(hist_data, var_name);
2684 }
2685
2686 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2687                                         char *system, char *event_name,
2688                                         char *var_name)
2689 {
2690         struct hist_field *var_field = NULL, *ref_field = NULL;
2691         struct trace_array *tr = hist_data->event_file->tr;
2692
2693         if (!is_var_ref(var_name))
2694                 return NULL;
2695
2696         var_name++;
2697
2698         var_field = find_event_var(hist_data, system, event_name, var_name);
2699         if (var_field)
2700                 ref_field = create_var_ref(hist_data, var_field,
2701                                            system, event_name);
2702
2703         if (!ref_field)
2704                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2705
2706         return ref_field;
2707 }
2708
2709 static struct ftrace_event_field *
2710 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2711             char *field_str, unsigned long *flags)
2712 {
2713         struct ftrace_event_field *field = NULL;
2714         char *field_name, *modifier, *str;
2715         struct trace_array *tr = file->tr;
2716
2717         modifier = str = kstrdup(field_str, GFP_KERNEL);
2718         if (!modifier)
2719                 return ERR_PTR(-ENOMEM);
2720
2721         field_name = strsep(&modifier, ".");
2722         if (modifier) {
2723                 if (strcmp(modifier, "hex") == 0)
2724                         *flags |= HIST_FIELD_FL_HEX;
2725                 else if (strcmp(modifier, "sym") == 0)
2726                         *flags |= HIST_FIELD_FL_SYM;
2727                 else if (strcmp(modifier, "sym-offset") == 0)
2728                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2729                 else if ((strcmp(modifier, "execname") == 0) &&
2730                          (strcmp(field_name, "common_pid") == 0))
2731                         *flags |= HIST_FIELD_FL_EXECNAME;
2732                 else if (strcmp(modifier, "syscall") == 0)
2733                         *flags |= HIST_FIELD_FL_SYSCALL;
2734                 else if (strcmp(modifier, "log2") == 0)
2735                         *flags |= HIST_FIELD_FL_LOG2;
2736                 else if (strcmp(modifier, "usecs") == 0)
2737                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2738                 else {
2739                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2740                         field = ERR_PTR(-EINVAL);
2741                         goto out;
2742                 }
2743         }
2744
2745         if (strcmp(field_name, "common_timestamp") == 0) {
2746                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2747                 hist_data->enable_timestamps = true;
2748                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2749                         hist_data->attrs->ts_in_usecs = true;
2750         } else if (strcmp(field_name, "cpu") == 0)
2751                 *flags |= HIST_FIELD_FL_CPU;
2752         else {
2753                 field = trace_find_event_field(file->event_call, field_name);
2754                 if (!field || !field->size) {
2755                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
2756                         field = ERR_PTR(-EINVAL);
2757                         goto out;
2758                 }
2759         }
2760  out:
2761         kfree(str);
2762
2763         return field;
2764 }
2765
2766 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2767                                        struct hist_field *var_ref,
2768                                        char *var_name)
2769 {
2770         struct hist_field *alias = NULL;
2771         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2772
2773         alias = create_hist_field(hist_data, NULL, flags, var_name);
2774         if (!alias)
2775                 return NULL;
2776
2777         alias->fn = var_ref->fn;
2778         alias->operands[0] = var_ref;
2779
2780         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2781                 destroy_hist_field(alias, 0);
2782                 return NULL;
2783         }
2784
2785         return alias;
2786 }
2787
2788 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2789                                      struct trace_event_file *file, char *str,
2790                                      unsigned long *flags, char *var_name)
2791 {
2792         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2793         struct ftrace_event_field *field = NULL;
2794         struct hist_field *hist_field = NULL;
2795         int ret = 0;
2796
2797         s = strchr(str, '.');
2798         if (s) {
2799                 s = strchr(++s, '.');
2800                 if (s) {
2801                         ref_system = strsep(&str, ".");
2802                         if (!str) {
2803                                 ret = -EINVAL;
2804                                 goto out;
2805                         }
2806                         ref_event = strsep(&str, ".");
2807                         if (!str) {
2808                                 ret = -EINVAL;
2809                                 goto out;
2810                         }
2811                         ref_var = str;
2812                 }
2813         }
2814
2815         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2816         if (!s) {
2817                 hist_field = parse_var_ref(hist_data, ref_system,
2818                                            ref_event, ref_var);
2819                 if (hist_field) {
2820                         if (var_name) {
2821                                 hist_field = create_alias(hist_data, hist_field, var_name);
2822                                 if (!hist_field) {
2823                                         ret = -ENOMEM;
2824                                         goto out;
2825                                 }
2826                         }
2827                         return hist_field;
2828                 }
2829         } else
2830                 str = s;
2831
2832         field = parse_field(hist_data, file, str, flags);
2833         if (IS_ERR(field)) {
2834                 ret = PTR_ERR(field);
2835                 goto out;
2836         }
2837
2838         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2839         if (!hist_field) {
2840                 ret = -ENOMEM;
2841                 goto out;
2842         }
2843
2844         return hist_field;
2845  out:
2846         return ERR_PTR(ret);
2847 }
2848
2849 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2850                                      struct trace_event_file *file,
2851                                      char *str, unsigned long flags,
2852                                      char *var_name, unsigned int level);
2853
2854 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2855                                       struct trace_event_file *file,
2856                                       char *str, unsigned long flags,
2857                                       char *var_name, unsigned int level)
2858 {
2859         struct hist_field *operand1, *expr = NULL;
2860         unsigned long operand_flags;
2861         int ret = 0;
2862         char *s;
2863
2864         /* we support only -(xxx) i.e. explicit parens required */
2865
2866         if (level > 3) {
2867                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2868                 ret = -EINVAL;
2869                 goto free;
2870         }
2871
2872         str++; /* skip leading '-' */
2873
2874         s = strchr(str, '(');
2875         if (s)
2876                 str++;
2877         else {
2878                 ret = -EINVAL;
2879                 goto free;
2880         }
2881
2882         s = strrchr(str, ')');
2883         if (s)
2884                 *s = '\0';
2885         else {
2886                 ret = -EINVAL; /* no closing ')' */
2887                 goto free;
2888         }
2889
2890         flags |= HIST_FIELD_FL_EXPR;
2891         expr = create_hist_field(hist_data, NULL, flags, var_name);
2892         if (!expr) {
2893                 ret = -ENOMEM;
2894                 goto free;
2895         }
2896
2897         operand_flags = 0;
2898         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2899         if (IS_ERR(operand1)) {
2900                 ret = PTR_ERR(operand1);
2901                 goto free;
2902         }
2903
2904         expr->flags |= operand1->flags &
2905                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2906         expr->fn = hist_field_unary_minus;
2907         expr->operands[0] = operand1;
2908         expr->operator = FIELD_OP_UNARY_MINUS;
2909         expr->name = expr_str(expr, 0);
2910         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2911         if (!expr->type) {
2912                 ret = -ENOMEM;
2913                 goto free;
2914         }
2915
2916         return expr;
2917  free:
2918         destroy_hist_field(expr, 0);
2919         return ERR_PTR(ret);
2920 }
2921
2922 static int check_expr_operands(struct trace_array *tr,
2923                                struct hist_field *operand1,
2924                                struct hist_field *operand2)
2925 {
2926         unsigned long operand1_flags = operand1->flags;
2927         unsigned long operand2_flags = operand2->flags;
2928
2929         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2930             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2931                 struct hist_field *var;
2932
2933                 var = find_var_field(operand1->var.hist_data, operand1->name);
2934                 if (!var)
2935                         return -EINVAL;
2936                 operand1_flags = var->flags;
2937         }
2938
2939         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2940             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2941                 struct hist_field *var;
2942
2943                 var = find_var_field(operand2->var.hist_data, operand2->name);
2944                 if (!var)
2945                         return -EINVAL;
2946                 operand2_flags = var->flags;
2947         }
2948
2949         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2950             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2951                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2952                 return -EINVAL;
2953         }
2954
2955         return 0;
2956 }
2957
2958 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2959                                      struct trace_event_file *file,
2960                                      char *str, unsigned long flags,
2961                                      char *var_name, unsigned int level)
2962 {
2963         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2964         unsigned long operand_flags;
2965         int field_op, ret = -EINVAL;
2966         char *sep, *operand1_str;
2967
2968         if (level > 3) {
2969                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2970                 return ERR_PTR(-EINVAL);
2971         }
2972
2973         field_op = contains_operator(str);
2974
2975         if (field_op == FIELD_OP_NONE)
2976                 return parse_atom(hist_data, file, str, &flags, var_name);
2977
2978         if (field_op == FIELD_OP_UNARY_MINUS)
2979                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2980
2981         switch (field_op) {
2982         case FIELD_OP_MINUS:
2983                 sep = "-";
2984                 break;
2985         case FIELD_OP_PLUS:
2986                 sep = "+";
2987                 break;
2988         default:
2989                 goto free;
2990         }
2991
2992         operand1_str = strsep(&str, sep);
2993         if (!operand1_str || !str)
2994                 goto free;
2995
2996         operand_flags = 0;
2997         operand1 = parse_atom(hist_data, file, operand1_str,
2998                               &operand_flags, NULL);
2999         if (IS_ERR(operand1)) {
3000                 ret = PTR_ERR(operand1);
3001                 operand1 = NULL;
3002                 goto free;
3003         }
3004
3005         /* rest of string could be another expression e.g. b+c in a+b+c */
3006         operand_flags = 0;
3007         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3008         if (IS_ERR(operand2)) {
3009                 ret = PTR_ERR(operand2);
3010                 operand2 = NULL;
3011                 goto free;
3012         }
3013
3014         ret = check_expr_operands(file->tr, operand1, operand2);
3015         if (ret)
3016                 goto free;
3017
3018         flags |= HIST_FIELD_FL_EXPR;
3019
3020         flags |= operand1->flags &
3021                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3022
3023         expr = create_hist_field(hist_data, NULL, flags, var_name);
3024         if (!expr) {
3025                 ret = -ENOMEM;
3026                 goto free;
3027         }
3028
3029         operand1->read_once = true;
3030         operand2->read_once = true;
3031
3032         expr->operands[0] = operand1;
3033         expr->operands[1] = operand2;
3034         expr->operator = field_op;
3035         expr->name = expr_str(expr, 0);
3036         expr->type = kstrdup(operand1->type, GFP_KERNEL);
3037         if (!expr->type) {
3038                 ret = -ENOMEM;
3039                 goto free;
3040         }
3041
3042         switch (field_op) {
3043         case FIELD_OP_MINUS:
3044                 expr->fn = hist_field_minus;
3045                 break;
3046         case FIELD_OP_PLUS:
3047                 expr->fn = hist_field_plus;
3048                 break;
3049         default:
3050                 ret = -EINVAL;
3051                 goto free;
3052         }
3053
3054         return expr;
3055  free:
3056         destroy_hist_field(operand1, 0);
3057         destroy_hist_field(operand2, 0);
3058         destroy_hist_field(expr, 0);
3059
3060         return ERR_PTR(ret);
3061 }
3062
3063 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
3064                                  struct trace_event_file *file)
3065 {
3066         struct event_trigger_data *test;
3067
3068         list_for_each_entry_rcu(test, &file->triggers, list) {
3069                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3070                         if (test->private_data == hist_data)
3071                                 return test->filter_str;
3072                 }
3073         }
3074
3075         return NULL;
3076 }
3077
3078 static struct event_command trigger_hist_cmd;
3079 static int event_hist_trigger_func(struct event_command *cmd_ops,
3080                                    struct trace_event_file *file,
3081                                    char *glob, char *cmd, char *param);
3082
3083 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
3084                             struct hist_trigger_data *hist_data,
3085                             unsigned int n_keys)
3086 {
3087         struct hist_field *target_hist_field, *hist_field;
3088         unsigned int n, i, j;
3089
3090         if (hist_data->n_fields - hist_data->n_vals != n_keys)
3091                 return false;
3092
3093         i = hist_data->n_vals;
3094         j = target_hist_data->n_vals;
3095
3096         for (n = 0; n < n_keys; n++) {
3097                 hist_field = hist_data->fields[i + n];
3098                 target_hist_field = target_hist_data->fields[j + n];
3099
3100                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
3101                         return false;
3102                 if (hist_field->size != target_hist_field->size)
3103                         return false;
3104                 if (hist_field->is_signed != target_hist_field->is_signed)
3105                         return false;
3106         }
3107
3108         return true;
3109 }
3110
3111 static struct hist_trigger_data *
3112 find_compatible_hist(struct hist_trigger_data *target_hist_data,
3113                      struct trace_event_file *file)
3114 {
3115         struct hist_trigger_data *hist_data;
3116         struct event_trigger_data *test;
3117         unsigned int n_keys;
3118
3119         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3120
3121         list_for_each_entry_rcu(test, &file->triggers, list) {
3122                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3123                         hist_data = test->private_data;
3124
3125                         if (compatible_keys(target_hist_data, hist_data, n_keys))
3126                                 return hist_data;
3127                 }
3128         }
3129
3130         return NULL;
3131 }
3132
3133 static struct trace_event_file *event_file(struct trace_array *tr,
3134                                            char *system, char *event_name)
3135 {
3136         struct trace_event_file *file;
3137
3138         file = __find_event_file(tr, system, event_name);
3139         if (!file)
3140                 return ERR_PTR(-EINVAL);
3141
3142         return file;
3143 }
3144
3145 static struct hist_field *
3146 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3147                          char *system, char *event_name, char *field_name)
3148 {
3149         struct hist_field *event_var;
3150         char *synthetic_name;
3151
3152         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3153         if (!synthetic_name)
3154                 return ERR_PTR(-ENOMEM);
3155
3156         strcpy(synthetic_name, "synthetic_");
3157         strcat(synthetic_name, field_name);
3158
3159         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3160
3161         kfree(synthetic_name);
3162
3163         return event_var;
3164 }
3165
3166 /**
3167  * create_field_var_hist - Automatically create a histogram and var for a field
3168  * @target_hist_data: The target hist trigger
3169  * @subsys_name: Optional subsystem name
3170  * @event_name: Optional event name
3171  * @field_name: The name of the field (and the resulting variable)
3172  *
3173  * Hist trigger actions fetch data from variables, not directly from
3174  * events.  However, for convenience, users are allowed to directly
3175  * specify an event field in an action, which will be automatically
3176  * converted into a variable on their behalf.
3177
3178  * If a user specifies a field on an event that isn't the event the
3179  * histogram currently being defined (the target event histogram), the
3180  * only way that can be accomplished is if a new hist trigger is
3181  * created and the field variable defined on that.
3182  *
3183  * This function creates a new histogram compatible with the target
3184  * event (meaning a histogram with the same key as the target
3185  * histogram), and creates a variable for the specified field, but
3186  * with 'synthetic_' prepended to the variable name in order to avoid
3187  * collision with normal field variables.
3188  *
3189  * Return: The variable created for the field.
3190  */
3191 static struct hist_field *
3192 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3193                       char *subsys_name, char *event_name, char *field_name)
3194 {
3195         struct trace_array *tr = target_hist_data->event_file->tr;
3196         struct hist_field *event_var = ERR_PTR(-EINVAL);
3197         struct hist_trigger_data *hist_data;
3198         unsigned int i, n, first = true;
3199         struct field_var_hist *var_hist;
3200         struct trace_event_file *file;
3201         struct hist_field *key_field;
3202         char *saved_filter;
3203         char *cmd;
3204         int ret;
3205
3206         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3207                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3208                 return ERR_PTR(-EINVAL);
3209         }
3210
3211         file = event_file(tr, subsys_name, event_name);
3212
3213         if (IS_ERR(file)) {
3214                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
3215                 ret = PTR_ERR(file);
3216                 return ERR_PTR(ret);
3217         }
3218
3219         /*
3220          * Look for a histogram compatible with target.  We'll use the
3221          * found histogram specification to create a new matching
3222          * histogram with our variable on it.  target_hist_data is not
3223          * yet a registered histogram so we can't use that.
3224          */
3225         hist_data = find_compatible_hist(target_hist_data, file);
3226         if (!hist_data) {
3227                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3228                 return ERR_PTR(-EINVAL);
3229         }
3230
3231         /* See if a synthetic field variable has already been created */
3232         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3233                                              event_name, field_name);
3234         if (!IS_ERR_OR_NULL(event_var))
3235                 return event_var;
3236
3237         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3238         if (!var_hist)
3239                 return ERR_PTR(-ENOMEM);
3240
3241         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3242         if (!cmd) {
3243                 kfree(var_hist);
3244                 return ERR_PTR(-ENOMEM);
3245         }
3246
3247         /* Use the same keys as the compatible histogram */
3248         strcat(cmd, "keys=");
3249
3250         for_each_hist_key_field(i, hist_data) {
3251                 key_field = hist_data->fields[i];
3252                 if (!first)
3253                         strcat(cmd, ",");
3254                 strcat(cmd, key_field->field->name);
3255                 first = false;
3256         }
3257
3258         /* Create the synthetic field variable specification */
3259         strcat(cmd, ":synthetic_");
3260         strcat(cmd, field_name);
3261         strcat(cmd, "=");
3262         strcat(cmd, field_name);
3263
3264         /* Use the same filter as the compatible histogram */
3265         saved_filter = find_trigger_filter(hist_data, file);
3266         if (saved_filter) {
3267                 strcat(cmd, " if ");
3268                 strcat(cmd, saved_filter);
3269         }
3270
3271         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3272         if (!var_hist->cmd) {
3273                 kfree(cmd);
3274                 kfree(var_hist);
3275                 return ERR_PTR(-ENOMEM);
3276         }
3277
3278         /* Save the compatible histogram information */
3279         var_hist->hist_data = hist_data;
3280
3281         /* Create the new histogram with our variable */
3282         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3283                                       "", "hist", cmd);
3284         if (ret) {
3285                 kfree(cmd);
3286                 kfree(var_hist->cmd);
3287                 kfree(var_hist);
3288                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3289                 return ERR_PTR(ret);
3290         }
3291
3292         kfree(cmd);
3293
3294         /* If we can't find the variable, something went wrong */
3295         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3296                                              event_name, field_name);
3297         if (IS_ERR_OR_NULL(event_var)) {
3298                 kfree(var_hist->cmd);
3299                 kfree(var_hist);
3300                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3301                 return ERR_PTR(-EINVAL);
3302         }
3303
3304         n = target_hist_data->n_field_var_hists;
3305         target_hist_data->field_var_hists[n] = var_hist;
3306         target_hist_data->n_field_var_hists++;
3307
3308         return event_var;
3309 }
3310
3311 static struct hist_field *
3312 find_target_event_var(struct hist_trigger_data *hist_data,
3313                       char *subsys_name, char *event_name, char *var_name)
3314 {
3315         struct trace_event_file *file = hist_data->event_file;
3316         struct hist_field *hist_field = NULL;
3317
3318         if (subsys_name) {
3319                 struct trace_event_call *call;
3320
3321                 if (!event_name)
3322                         return NULL;
3323
3324                 call = file->event_call;
3325
3326                 if (strcmp(subsys_name, call->class->system) != 0)
3327                         return NULL;
3328
3329                 if (strcmp(event_name, trace_event_name(call)) != 0)
3330                         return NULL;
3331         }
3332
3333         hist_field = find_var_field(hist_data, var_name);
3334
3335         return hist_field;
3336 }
3337
3338 static inline void __update_field_vars(struct tracing_map_elt *elt,
3339                                        struct ring_buffer_event *rbe,
3340                                        void *rec,
3341                                        struct field_var **field_vars,
3342                                        unsigned int n_field_vars,
3343                                        unsigned int field_var_str_start)
3344 {
3345         struct hist_elt_data *elt_data = elt->private_data;
3346         unsigned int i, j, var_idx;
3347         u64 var_val;
3348
3349         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3350                 struct field_var *field_var = field_vars[i];
3351                 struct hist_field *var = field_var->var;
3352                 struct hist_field *val = field_var->val;
3353
3354                 var_val = val->fn(val, elt, rbe, rec);
3355                 var_idx = var->var.idx;
3356
3357                 if (val->flags & HIST_FIELD_FL_STRING) {
3358                         char *str = elt_data->field_var_str[j++];
3359                         char *val_str = (char *)(uintptr_t)var_val;
3360
3361                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3362                         var_val = (u64)(uintptr_t)str;
3363                 }
3364                 tracing_map_set_var(elt, var_idx, var_val);
3365         }
3366 }
3367
3368 static void update_field_vars(struct hist_trigger_data *hist_data,
3369                               struct tracing_map_elt *elt,
3370                               struct ring_buffer_event *rbe,
3371                               void *rec)
3372 {
3373         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3374                             hist_data->n_field_vars, 0);
3375 }
3376
3377 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3378                                  struct tracing_map_elt *elt, void *rec,
3379                                  struct ring_buffer_event *rbe, void *key,
3380                                  struct action_data *data, u64 *var_ref_vals)
3381 {
3382         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
3383                             hist_data->n_save_vars, hist_data->n_field_var_str);
3384 }
3385
3386 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3387                                      struct trace_event_file *file,
3388                                      char *name, int size, const char *type)
3389 {
3390         struct hist_field *var;
3391         int idx;
3392
3393         if (find_var(hist_data, file, name) && !hist_data->remove) {
3394                 var = ERR_PTR(-EINVAL);
3395                 goto out;
3396         }
3397
3398         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3399         if (!var) {
3400                 var = ERR_PTR(-ENOMEM);
3401                 goto out;
3402         }
3403
3404         idx = tracing_map_add_var(hist_data->map);
3405         if (idx < 0) {
3406                 kfree(var);
3407                 var = ERR_PTR(-EINVAL);
3408                 goto out;
3409         }
3410
3411         var->flags = HIST_FIELD_FL_VAR;
3412         var->var.idx = idx;
3413         var->var.hist_data = var->hist_data = hist_data;
3414         var->size = size;
3415         var->var.name = kstrdup(name, GFP_KERNEL);
3416         var->type = kstrdup(type, GFP_KERNEL);
3417         if (!var->var.name || !var->type) {
3418                 kfree(var->var.name);
3419                 kfree(var->type);
3420                 kfree(var);
3421                 var = ERR_PTR(-ENOMEM);
3422         }
3423  out:
3424         return var;
3425 }
3426
3427 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3428                                           struct trace_event_file *file,
3429                                           char *field_name)
3430 {
3431         struct hist_field *val = NULL, *var = NULL;
3432         unsigned long flags = HIST_FIELD_FL_VAR;
3433         struct trace_array *tr = file->tr;
3434         struct field_var *field_var;
3435         int ret = 0;
3436
3437         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3438                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3439                 ret = -EINVAL;
3440                 goto err;
3441         }
3442
3443         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3444         if (IS_ERR(val)) {
3445                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3446                 ret = PTR_ERR(val);
3447                 goto err;
3448         }
3449
3450         var = create_var(hist_data, file, field_name, val->size, val->type);
3451         if (IS_ERR(var)) {
3452                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3453                 kfree(val);
3454                 ret = PTR_ERR(var);
3455                 goto err;
3456         }
3457
3458         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3459         if (!field_var) {
3460                 kfree(val);
3461                 kfree(var);
3462                 ret =  -ENOMEM;
3463                 goto err;
3464         }
3465
3466         field_var->var = var;
3467         field_var->val = val;
3468  out:
3469         return field_var;
3470  err:
3471         field_var = ERR_PTR(ret);
3472         goto out;
3473 }
3474
3475 /**
3476  * create_target_field_var - Automatically create a variable for a field
3477  * @target_hist_data: The target hist trigger
3478  * @subsys_name: Optional subsystem name
3479  * @event_name: Optional event name
3480  * @var_name: The name of the field (and the resulting variable)
3481  *
3482  * Hist trigger actions fetch data from variables, not directly from
3483  * events.  However, for convenience, users are allowed to directly
3484  * specify an event field in an action, which will be automatically
3485  * converted into a variable on their behalf.
3486
3487  * This function creates a field variable with the name var_name on
3488  * the hist trigger currently being defined on the target event.  If
3489  * subsys_name and event_name are specified, this function simply
3490  * verifies that they do in fact match the target event subsystem and
3491  * event name.
3492  *
3493  * Return: The variable created for the field.
3494  */
3495 static struct field_var *
3496 create_target_field_var(struct hist_trigger_data *target_hist_data,
3497                         char *subsys_name, char *event_name, char *var_name)
3498 {
3499         struct trace_event_file *file = target_hist_data->event_file;
3500
3501         if (subsys_name) {
3502                 struct trace_event_call *call;
3503
3504                 if (!event_name)
3505                         return NULL;
3506
3507                 call = file->event_call;
3508
3509                 if (strcmp(subsys_name, call->class->system) != 0)
3510                         return NULL;
3511
3512                 if (strcmp(event_name, trace_event_name(call)) != 0)
3513                         return NULL;
3514         }
3515
3516         return create_field_var(target_hist_data, file, var_name);
3517 }
3518
3519 static bool check_track_val_max(u64 track_val, u64 var_val)
3520 {
3521         if (var_val <= track_val)
3522                 return false;
3523
3524         return true;
3525 }
3526
3527 static bool check_track_val_changed(u64 track_val, u64 var_val)
3528 {
3529         if (var_val == track_val)
3530                 return false;
3531
3532         return true;
3533 }
3534
3535 static u64 get_track_val(struct hist_trigger_data *hist_data,
3536                          struct tracing_map_elt *elt,
3537                          struct action_data *data)
3538 {
3539         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3540         u64 track_val;
3541
3542         track_val = tracing_map_read_var(elt, track_var_idx);
3543
3544         return track_val;
3545 }
3546
3547 static void save_track_val(struct hist_trigger_data *hist_data,
3548                            struct tracing_map_elt *elt,
3549                            struct action_data *data, u64 var_val)
3550 {
3551         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3552
3553         tracing_map_set_var(elt, track_var_idx, var_val);
3554 }
3555
3556 static void save_track_data(struct hist_trigger_data *hist_data,
3557                             struct tracing_map_elt *elt, void *rec,
3558                             struct ring_buffer_event *rbe, void *key,
3559                             struct action_data *data, u64 *var_ref_vals)
3560 {
3561         if (data->track_data.save_data)
3562                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3563 }
3564
3565 static bool check_track_val(struct tracing_map_elt *elt,
3566                             struct action_data *data,
3567                             u64 var_val)
3568 {
3569         struct hist_trigger_data *hist_data;
3570         u64 track_val;
3571
3572         hist_data = data->track_data.track_var->hist_data;
3573         track_val = get_track_val(hist_data, elt, data);
3574
3575         return data->track_data.check_val(track_val, var_val);
3576 }
3577
3578 #ifdef CONFIG_TRACER_SNAPSHOT
3579 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3580 {
3581         /* called with tr->max_lock held */
3582         struct track_data *track_data = tr->cond_snapshot->cond_data;
3583         struct hist_elt_data *elt_data, *track_elt_data;
3584         struct snapshot_context *context = cond_data;
3585         u64 track_val;
3586
3587         if (!track_data)
3588                 return false;
3589
3590         track_val = get_track_val(track_data->hist_data, context->elt,
3591                                   track_data->action_data);
3592
3593         track_data->track_val = track_val;
3594         memcpy(track_data->key, context->key, track_data->key_len);
3595
3596         elt_data = context->elt->private_data;
3597         track_elt_data = track_data->elt.private_data;
3598         if (elt_data->comm)
3599                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3600
3601         track_data->updated = true;
3602
3603         return true;
3604 }
3605
3606 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3607                                      struct tracing_map_elt *elt, void *rec,
3608                                      struct ring_buffer_event *rbe, void *key,
3609                                      struct action_data *data,
3610                                      u64 *var_ref_vals)
3611 {
3612         struct trace_event_file *file = hist_data->event_file;
3613         struct snapshot_context context;
3614
3615         context.elt = elt;
3616         context.key = key;
3617
3618         tracing_snapshot_cond(file->tr, &context);
3619 }
3620
3621 static void hist_trigger_print_key(struct seq_file *m,
3622                                    struct hist_trigger_data *hist_data,
3623                                    void *key,
3624                                    struct tracing_map_elt *elt);
3625
3626 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3627 {
3628         unsigned int i;
3629
3630         if (!hist_data->n_actions)
3631                 return NULL;
3632
3633         for (i = 0; i < hist_data->n_actions; i++) {
3634                 struct action_data *data = hist_data->actions[i];
3635
3636                 if (data->action == ACTION_SNAPSHOT)
3637                         return data;
3638         }
3639
3640         return NULL;
3641 }
3642
3643 static void track_data_snapshot_print(struct seq_file *m,
3644                                       struct hist_trigger_data *hist_data)
3645 {
3646         struct trace_event_file *file = hist_data->event_file;
3647         struct track_data *track_data;
3648         struct action_data *action;
3649
3650         track_data = tracing_cond_snapshot_data(file->tr);
3651         if (!track_data)
3652                 return;
3653
3654         if (!track_data->updated)
3655                 return;
3656
3657         action = snapshot_action(hist_data);
3658         if (!action)
3659                 return;
3660
3661         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3662         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3663                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3664                    action->track_data.var_str, track_data->track_val);
3665
3666         seq_puts(m, "\ttriggered by event with key: ");
3667         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3668         seq_putc(m, '\n');
3669 }
3670 #else
3671 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3672 {
3673         return false;
3674 }
3675 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3676                                      struct tracing_map_elt *elt, void *rec,
3677                                      struct ring_buffer_event *rbe, void *key,
3678                                      struct action_data *data,
3679                                      u64 *var_ref_vals) {}
3680 static void track_data_snapshot_print(struct seq_file *m,
3681                                       struct hist_trigger_data *hist_data) {}
3682 #endif /* CONFIG_TRACER_SNAPSHOT */
3683
3684 static void track_data_print(struct seq_file *m,
3685                              struct hist_trigger_data *hist_data,
3686                              struct tracing_map_elt *elt,
3687                              struct action_data *data)
3688 {
3689         u64 track_val = get_track_val(hist_data, elt, data);
3690         unsigned int i, save_var_idx;
3691
3692         if (data->handler == HANDLER_ONMAX)
3693                 seq_printf(m, "\n\tmax: %10llu", track_val);
3694         else if (data->handler == HANDLER_ONCHANGE)
3695                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3696
3697         if (data->action == ACTION_SNAPSHOT)
3698                 return;
3699
3700         for (i = 0; i < hist_data->n_save_vars; i++) {
3701                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3702                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3703                 u64 val;
3704
3705                 save_var_idx = save_var->var.idx;
3706
3707                 val = tracing_map_read_var(elt, save_var_idx);
3708
3709                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3710                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3711                                    (char *)(uintptr_t)(val));
3712                 } else
3713                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3714         }
3715 }
3716
3717 static void ontrack_action(struct hist_trigger_data *hist_data,
3718                            struct tracing_map_elt *elt, void *rec,
3719                            struct ring_buffer_event *rbe, void *key,
3720                            struct action_data *data, u64 *var_ref_vals)
3721 {
3722         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3723
3724         if (check_track_val(elt, data, var_val)) {
3725                 save_track_val(hist_data, elt, data, var_val);
3726                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3727         }
3728 }
3729
3730 static void action_data_destroy(struct action_data *data)
3731 {
3732         unsigned int i;
3733
3734         lockdep_assert_held(&event_mutex);
3735
3736         kfree(data->action_name);
3737
3738         for (i = 0; i < data->n_params; i++)
3739                 kfree(data->params[i]);
3740
3741         if (data->synth_event)
3742                 data->synth_event->ref--;
3743
3744         kfree(data->synth_event_name);
3745
3746         kfree(data);
3747 }
3748
3749 static void track_data_destroy(struct hist_trigger_data *hist_data,
3750                                struct action_data *data)
3751 {
3752         struct trace_event_file *file = hist_data->event_file;
3753
3754         destroy_hist_field(data->track_data.track_var, 0);
3755
3756         if (data->action == ACTION_SNAPSHOT) {
3757                 struct track_data *track_data;
3758
3759                 track_data = tracing_cond_snapshot_data(file->tr);
3760                 if (track_data && track_data->hist_data == hist_data) {
3761                         tracing_snapshot_cond_disable(file->tr);
3762                         track_data_free(track_data);
3763                 }
3764         }
3765
3766         kfree(data->track_data.var_str);
3767
3768         action_data_destroy(data);
3769 }
3770
3771 static int action_create(struct hist_trigger_data *hist_data,
3772                          struct action_data *data);
3773
3774 static int track_data_create(struct hist_trigger_data *hist_data,
3775                              struct action_data *data)
3776 {
3777         struct hist_field *var_field, *ref_field, *track_var = NULL;
3778         struct trace_event_file *file = hist_data->event_file;
3779         struct trace_array *tr = file->tr;
3780         char *track_data_var_str;
3781         int ret = 0;
3782
3783         track_data_var_str = data->track_data.var_str;
3784         if (track_data_var_str[0] != '$') {
3785                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3786                 return -EINVAL;
3787         }
3788         track_data_var_str++;
3789
3790         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3791         if (!var_field) {
3792                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3793                 return -EINVAL;
3794         }
3795
3796         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3797         if (!ref_field)
3798                 return -ENOMEM;
3799
3800         data->track_data.var_ref = ref_field;
3801
3802         if (data->handler == HANDLER_ONMAX)
3803                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3804         if (IS_ERR(track_var)) {
3805                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3806                 ret = PTR_ERR(track_var);
3807                 goto out;
3808         }
3809
3810         if (data->handler == HANDLER_ONCHANGE)
3811                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3812         if (IS_ERR(track_var)) {
3813                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3814                 ret = PTR_ERR(track_var);
3815                 goto out;
3816         }
3817         data->track_data.track_var = track_var;
3818
3819         ret = action_create(hist_data, data);
3820  out:
3821         return ret;
3822 }
3823
3824 static int parse_action_params(struct trace_array *tr, char *params,
3825                                struct action_data *data)
3826 {
3827         char *param, *saved_param;
3828         bool first_param = true;
3829         int ret = 0;
3830
3831         while (params) {
3832                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3833                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3834                         goto out;
3835                 }
3836
3837                 param = strsep(&params, ",");
3838                 if (!param) {
3839                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3840                         ret = -EINVAL;
3841                         goto out;
3842                 }
3843
3844                 param = strstrip(param);
3845                 if (strlen(param) < 2) {
3846                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3847                         ret = -EINVAL;
3848                         goto out;
3849                 }
3850
3851                 saved_param = kstrdup(param, GFP_KERNEL);
3852                 if (!saved_param) {
3853                         ret = -ENOMEM;
3854                         goto out;
3855                 }
3856
3857                 if (first_param && data->use_trace_keyword) {
3858                         data->synth_event_name = saved_param;
3859                         first_param = false;
3860                         continue;
3861                 }
3862                 first_param = false;
3863
3864                 data->params[data->n_params++] = saved_param;
3865         }
3866  out:
3867         return ret;
3868 }
3869
3870 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3871                         enum handler_id handler)
3872 {
3873         char *action_name;
3874         int ret = 0;
3875
3876         strsep(&str, ".");
3877         if (!str) {
3878                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3879                 ret = -EINVAL;
3880                 goto out;
3881         }
3882
3883         action_name = strsep(&str, "(");
3884         if (!action_name || !str) {
3885                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3886                 ret = -EINVAL;
3887                 goto out;
3888         }
3889
3890         if (str_has_prefix(action_name, "save")) {
3891                 char *params = strsep(&str, ")");
3892
3893                 if (!params) {
3894                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3895                         ret = -EINVAL;
3896                         goto out;
3897                 }
3898
3899                 ret = parse_action_params(tr, params, data);
3900                 if (ret)
3901                         goto out;
3902
3903                 if (handler == HANDLER_ONMAX)
3904                         data->track_data.check_val = check_track_val_max;
3905                 else if (handler == HANDLER_ONCHANGE)
3906                         data->track_data.check_val = check_track_val_changed;
3907                 else {
3908                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3909                         ret = -EINVAL;
3910                         goto out;
3911                 }
3912
3913                 data->track_data.save_data = save_track_data_vars;
3914                 data->fn = ontrack_action;
3915                 data->action = ACTION_SAVE;
3916         } else if (str_has_prefix(action_name, "snapshot")) {
3917                 char *params = strsep(&str, ")");
3918
3919                 if (!str) {
3920                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3921                         ret = -EINVAL;
3922                         goto out;
3923                 }
3924
3925                 if (handler == HANDLER_ONMAX)
3926                         data->track_data.check_val = check_track_val_max;
3927                 else if (handler == HANDLER_ONCHANGE)
3928                         data->track_data.check_val = check_track_val_changed;
3929                 else {
3930                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3931                         ret = -EINVAL;
3932                         goto out;
3933                 }
3934
3935                 data->track_data.save_data = save_track_data_snapshot;
3936                 data->fn = ontrack_action;
3937                 data->action = ACTION_SNAPSHOT;
3938         } else {
3939                 char *params = strsep(&str, ")");
3940
3941                 if (str_has_prefix(action_name, "trace"))
3942                         data->use_trace_keyword = true;
3943
3944                 if (params) {
3945                         ret = parse_action_params(tr, params, data);
3946                         if (ret)
3947                                 goto out;
3948                 }
3949
3950                 if (handler == HANDLER_ONMAX)
3951                         data->track_data.check_val = check_track_val_max;
3952                 else if (handler == HANDLER_ONCHANGE)
3953                         data->track_data.check_val = check_track_val_changed;
3954
3955                 if (handler != HANDLER_ONMATCH) {
3956                         data->track_data.save_data = action_trace;
3957                         data->fn = ontrack_action;
3958                 } else
3959                         data->fn = action_trace;
3960
3961                 data->action = ACTION_TRACE;
3962         }
3963
3964         data->action_name = kstrdup(action_name, GFP_KERNEL);
3965         if (!data->action_name) {
3966                 ret = -ENOMEM;
3967                 goto out;
3968         }
3969
3970         data->handler = handler;
3971  out:
3972         return ret;
3973 }
3974
3975 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3976                                             char *str, enum handler_id handler)
3977 {
3978         struct action_data *data;
3979         int ret = -EINVAL;
3980         char *var_str;
3981
3982         data = kzalloc(sizeof(*data), GFP_KERNEL);
3983         if (!data)
3984                 return ERR_PTR(-ENOMEM);
3985
3986         var_str = strsep(&str, ")");
3987         if (!var_str || !str) {
3988                 ret = -EINVAL;
3989                 goto free;
3990         }
3991
3992         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3993         if (!data->track_data.var_str) {
3994                 ret = -ENOMEM;
3995                 goto free;
3996         }
3997
3998         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3999         if (ret)
4000                 goto free;
4001  out:
4002         return data;
4003  free:
4004         track_data_destroy(hist_data, data);
4005         data = ERR_PTR(ret);
4006         goto out;
4007 }
4008
4009 static void onmatch_destroy(struct action_data *data)
4010 {
4011         kfree(data->match_data.event);
4012         kfree(data->match_data.event_system);
4013
4014         action_data_destroy(data);
4015 }
4016
4017 static void destroy_field_var(struct field_var *field_var)
4018 {
4019         if (!field_var)
4020                 return;
4021
4022         destroy_hist_field(field_var->var, 0);
4023         destroy_hist_field(field_var->val, 0);
4024
4025         kfree(field_var);
4026 }
4027
4028 static void destroy_field_vars(struct hist_trigger_data *hist_data)
4029 {
4030         unsigned int i;
4031
4032         for (i = 0; i < hist_data->n_field_vars; i++)
4033                 destroy_field_var(hist_data->field_vars[i]);
4034 }
4035
4036 static void save_field_var(struct hist_trigger_data *hist_data,
4037                            struct field_var *field_var)
4038 {
4039         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
4040
4041         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4042                 hist_data->n_field_var_str++;
4043 }
4044
4045
4046 static int check_synth_field(struct synth_event *event,
4047                              struct hist_field *hist_field,
4048                              unsigned int field_pos)
4049 {
4050         struct synth_field *field;
4051
4052         if (field_pos >= event->n_fields)
4053                 return -EINVAL;
4054
4055         field = event->fields[field_pos];
4056
4057         if (strcmp(field->type, hist_field->type) != 0)
4058                 return -EINVAL;
4059
4060         return 0;
4061 }
4062
4063 static struct hist_field *
4064 trace_action_find_var(struct hist_trigger_data *hist_data,
4065                       struct action_data *data,
4066                       char *system, char *event, char *var)
4067 {
4068         struct trace_array *tr = hist_data->event_file->tr;
4069         struct hist_field *hist_field;
4070
4071         var++; /* skip '$' */
4072
4073         hist_field = find_target_event_var(hist_data, system, event, var);
4074         if (!hist_field) {
4075                 if (!system && data->handler == HANDLER_ONMATCH) {
4076                         system = data->match_data.event_system;
4077                         event = data->match_data.event;
4078                 }
4079
4080                 hist_field = find_event_var(hist_data, system, event, var);
4081         }
4082
4083         if (!hist_field)
4084                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
4085
4086         return hist_field;
4087 }
4088
4089 static struct hist_field *
4090 trace_action_create_field_var(struct hist_trigger_data *hist_data,
4091                               struct action_data *data, char *system,
4092                               char *event, char *var)
4093 {
4094         struct hist_field *hist_field = NULL;
4095         struct field_var *field_var;
4096
4097         /*
4098          * First try to create a field var on the target event (the
4099          * currently being defined).  This will create a variable for
4100          * unqualified fields on the target event, or if qualified,
4101          * target fields that have qualified names matching the target.
4102          */
4103         field_var = create_target_field_var(hist_data, system, event, var);
4104
4105         if (field_var && !IS_ERR(field_var)) {
4106                 save_field_var(hist_data, field_var);
4107                 hist_field = field_var->var;
4108         } else {
4109                 field_var = NULL;
4110                 /*
4111                  * If no explicit system.event is specfied, default to
4112                  * looking for fields on the onmatch(system.event.xxx)
4113                  * event.
4114                  */
4115                 if (!system && data->handler == HANDLER_ONMATCH) {
4116                         system = data->match_data.event_system;
4117                         event = data->match_data.event;
4118                 }
4119
4120                 /*
4121                  * At this point, we're looking at a field on another
4122                  * event.  Because we can't modify a hist trigger on
4123                  * another event to add a variable for a field, we need
4124                  * to create a new trigger on that event and create the
4125                  * variable at the same time.
4126                  */
4127                 hist_field = create_field_var_hist(hist_data, system, event, var);
4128                 if (IS_ERR(hist_field))
4129                         goto free;
4130         }
4131  out:
4132         return hist_field;
4133  free:
4134         destroy_field_var(field_var);
4135         hist_field = NULL;
4136         goto out;
4137 }
4138
4139 static int trace_action_create(struct hist_trigger_data *hist_data,
4140                                struct action_data *data)
4141 {
4142         struct trace_array *tr = hist_data->event_file->tr;
4143         char *event_name, *param, *system = NULL;
4144         struct hist_field *hist_field, *var_ref;
4145         unsigned int i, var_ref_idx;
4146         unsigned int field_pos = 0;
4147         struct synth_event *event;
4148         char *synth_event_name;
4149         int ret = 0;
4150
4151         lockdep_assert_held(&event_mutex);
4152
4153         if (data->use_trace_keyword)
4154                 synth_event_name = data->synth_event_name;
4155         else
4156                 synth_event_name = data->action_name;
4157
4158         event = find_synth_event(synth_event_name);
4159         if (!event) {
4160                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4161                 return -EINVAL;
4162         }
4163
4164         event->ref++;
4165
4166         var_ref_idx = hist_data->n_var_refs;
4167
4168         for (i = 0; i < data->n_params; i++) {
4169                 char *p;
4170
4171                 p = param = kstrdup(data->params[i], GFP_KERNEL);
4172                 if (!param) {
4173                         ret = -ENOMEM;
4174                         goto err;
4175                 }
4176
4177                 system = strsep(&param, ".");
4178                 if (!param) {
4179                         param = (char *)system;
4180                         system = event_name = NULL;
4181                 } else {
4182                         event_name = strsep(&param, ".");
4183                         if (!param) {
4184                                 kfree(p);
4185                                 ret = -EINVAL;
4186                                 goto err;
4187                         }
4188                 }
4189
4190                 if (param[0] == '$')
4191                         hist_field = trace_action_find_var(hist_data, data,
4192                                                            system, event_name,
4193                                                            param);
4194                 else
4195                         hist_field = trace_action_create_field_var(hist_data,
4196                                                                    data,
4197                                                                    system,
4198                                                                    event_name,
4199                                                                    param);
4200
4201                 if (!hist_field) {
4202                         kfree(p);
4203                         ret = -EINVAL;
4204                         goto err;
4205                 }
4206
4207                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4208                         var_ref = create_var_ref(hist_data, hist_field,
4209                                                  system, event_name);
4210                         if (!var_ref) {
4211                                 kfree(p);
4212                                 ret = -ENOMEM;
4213                                 goto err;
4214                         }
4215
4216                         field_pos++;
4217                         kfree(p);
4218                         continue;
4219                 }
4220
4221                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4222                 kfree(p);
4223                 ret = -EINVAL;
4224                 goto err;
4225         }
4226
4227         if (field_pos != event->n_fields) {
4228                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4229                 ret = -EINVAL;
4230                 goto err;
4231         }
4232
4233         data->synth_event = event;
4234         data->var_ref_idx = var_ref_idx;
4235  out:
4236         return ret;
4237  err:
4238         event->ref--;
4239
4240         goto out;
4241 }
4242
4243 static int action_create(struct hist_trigger_data *hist_data,
4244                          struct action_data *data)
4245 {
4246         struct trace_event_file *file = hist_data->event_file;
4247         struct trace_array *tr = file->tr;
4248         struct track_data *track_data;
4249         struct field_var *field_var;
4250         unsigned int i;
4251         char *param;
4252         int ret = 0;
4253
4254         if (data->action == ACTION_TRACE)
4255                 return trace_action_create(hist_data, data);
4256
4257         if (data->action == ACTION_SNAPSHOT) {
4258                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4259                 if (IS_ERR(track_data)) {
4260                         ret = PTR_ERR(track_data);
4261                         goto out;
4262                 }
4263
4264                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4265                                                    cond_snapshot_update);
4266                 if (ret)
4267                         track_data_free(track_data);
4268
4269                 goto out;
4270         }
4271
4272         if (data->action == ACTION_SAVE) {
4273                 if (hist_data->n_save_vars) {
4274                         ret = -EEXIST;
4275                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4276                         goto out;
4277                 }
4278
4279                 for (i = 0; i < data->n_params; i++) {
4280                         param = kstrdup(data->params[i], GFP_KERNEL);
4281                         if (!param) {
4282                                 ret = -ENOMEM;
4283                                 goto out;
4284                         }
4285
4286                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4287                         if (IS_ERR(field_var)) {
4288                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4289                                          errpos(param));
4290                                 ret = PTR_ERR(field_var);
4291                                 kfree(param);
4292                                 goto out;
4293                         }
4294
4295                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4296                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4297                                 hist_data->n_save_var_str++;
4298                         kfree(param);
4299                 }
4300         }
4301  out:
4302         return ret;
4303 }
4304
4305 static int onmatch_create(struct hist_trigger_data *hist_data,
4306                           struct action_data *data)
4307 {
4308         return action_create(hist_data, data);
4309 }
4310
4311 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4312 {
4313         char *match_event, *match_event_system;
4314         struct action_data *data;
4315         int ret = -EINVAL;
4316
4317         data = kzalloc(sizeof(*data), GFP_KERNEL);
4318         if (!data)
4319                 return ERR_PTR(-ENOMEM);
4320
4321         match_event = strsep(&str, ")");
4322         if (!match_event || !str) {
4323                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4324                 goto free;
4325         }
4326
4327         match_event_system = strsep(&match_event, ".");
4328         if (!match_event) {
4329                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4330                 goto free;
4331         }
4332
4333         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4334                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4335                 goto free;
4336         }
4337
4338         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4339         if (!data->match_data.event) {
4340                 ret = -ENOMEM;
4341                 goto free;
4342         }
4343
4344         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4345         if (!data->match_data.event_system) {
4346                 ret = -ENOMEM;
4347                 goto free;
4348         }
4349
4350         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4351         if (ret)
4352                 goto free;
4353  out:
4354         return data;
4355  free:
4356         onmatch_destroy(data);
4357         data = ERR_PTR(ret);
4358         goto out;
4359 }
4360
4361 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4362 {
4363         hist_data->fields[HITCOUNT_IDX] =
4364                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4365         if (!hist_data->fields[HITCOUNT_IDX])
4366                 return -ENOMEM;
4367
4368         hist_data->n_vals++;
4369         hist_data->n_fields++;
4370
4371         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4372                 return -EINVAL;
4373
4374         return 0;
4375 }
4376
4377 static int __create_val_field(struct hist_trigger_data *hist_data,
4378                               unsigned int val_idx,
4379                               struct trace_event_file *file,
4380                               char *var_name, char *field_str,
4381                               unsigned long flags)
4382 {
4383         struct hist_field *hist_field;
4384         int ret = 0;
4385
4386         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
4387         if (IS_ERR(hist_field)) {
4388                 ret = PTR_ERR(hist_field);
4389                 goto out;
4390         }
4391
4392         hist_data->fields[val_idx] = hist_field;
4393
4394         ++hist_data->n_vals;
4395         ++hist_data->n_fields;
4396
4397         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4398                 ret = -EINVAL;
4399  out:
4400         return ret;
4401 }
4402
4403 static int create_val_field(struct hist_trigger_data *hist_data,
4404                             unsigned int val_idx,
4405                             struct trace_event_file *file,
4406                             char *field_str)
4407 {
4408         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4409                 return -EINVAL;
4410
4411         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4412 }
4413
4414 static int create_var_field(struct hist_trigger_data *hist_data,
4415                             unsigned int val_idx,
4416                             struct trace_event_file *file,
4417                             char *var_name, char *expr_str)
4418 {
4419         struct trace_array *tr = hist_data->event_file->tr;
4420         unsigned long flags = 0;
4421
4422         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4423                 return -EINVAL;
4424
4425         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4426                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4427                 return -EINVAL;
4428         }
4429
4430         flags |= HIST_FIELD_FL_VAR;
4431         hist_data->n_vars++;
4432         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4433                 return -EINVAL;
4434
4435         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4436 }
4437
4438 static int create_val_fields(struct hist_trigger_data *hist_data,
4439                              struct trace_event_file *file)
4440 {
4441         char *fields_str, *field_str;
4442         unsigned int i, j = 1;
4443         int ret;
4444
4445         ret = create_hitcount_val(hist_data);
4446         if (ret)
4447                 goto out;
4448
4449         fields_str = hist_data->attrs->vals_str;
4450         if (!fields_str)
4451                 goto out;
4452
4453         strsep(&fields_str, "=");
4454         if (!fields_str)
4455                 goto out;
4456
4457         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4458                      j < TRACING_MAP_VALS_MAX; i++) {
4459                 field_str = strsep(&fields_str, ",");
4460                 if (!field_str)
4461                         break;
4462
4463                 if (strcmp(field_str, "hitcount") == 0)
4464                         continue;
4465
4466                 ret = create_val_field(hist_data, j++, file, field_str);
4467                 if (ret)
4468                         goto out;
4469         }
4470
4471         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4472                 ret = -EINVAL;
4473  out:
4474         return ret;
4475 }
4476
4477 static int create_key_field(struct hist_trigger_data *hist_data,
4478                             unsigned int key_idx,
4479                             unsigned int key_offset,
4480                             struct trace_event_file *file,
4481                             char *field_str)
4482 {
4483         struct trace_array *tr = hist_data->event_file->tr;
4484         struct hist_field *hist_field = NULL;
4485         unsigned long flags = 0;
4486         unsigned int key_size;
4487         int ret = 0;
4488
4489         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4490                 return -EINVAL;
4491
4492         flags |= HIST_FIELD_FL_KEY;
4493
4494         if (strcmp(field_str, "stacktrace") == 0) {
4495                 flags |= HIST_FIELD_FL_STACKTRACE;
4496                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4497                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4498         } else {
4499                 hist_field = parse_expr(hist_data, file, field_str, flags,
4500                                         NULL, 0);
4501                 if (IS_ERR(hist_field)) {
4502                         ret = PTR_ERR(hist_field);
4503                         goto out;
4504                 }
4505
4506                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
4507                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4508                         destroy_hist_field(hist_field, 0);
4509                         ret = -EINVAL;
4510                         goto out;
4511                 }
4512
4513                 key_size = hist_field->size;
4514         }
4515
4516         hist_data->fields[key_idx] = hist_field;
4517
4518         key_size = ALIGN(key_size, sizeof(u64));
4519         hist_data->fields[key_idx]->size = key_size;
4520         hist_data->fields[key_idx]->offset = key_offset;
4521
4522         hist_data->key_size += key_size;
4523
4524         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4525                 ret = -EINVAL;
4526                 goto out;
4527         }
4528
4529         hist_data->n_keys++;
4530         hist_data->n_fields++;
4531
4532         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4533                 return -EINVAL;
4534
4535         ret = key_size;
4536  out:
4537         return ret;
4538 }
4539
4540 static int create_key_fields(struct hist_trigger_data *hist_data,
4541                              struct trace_event_file *file)
4542 {
4543         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4544         char *fields_str, *field_str;
4545         int ret = -EINVAL;
4546
4547         fields_str = hist_data->attrs->keys_str;
4548         if (!fields_str)
4549                 goto out;
4550
4551         strsep(&fields_str, "=");
4552         if (!fields_str)
4553                 goto out;
4554
4555         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4556                 field_str = strsep(&fields_str, ",");
4557                 if (!field_str)
4558                         break;
4559                 ret = create_key_field(hist_data, i, key_offset,
4560                                        file, field_str);
4561                 if (ret < 0)
4562                         goto out;
4563                 key_offset += ret;
4564         }
4565         if (fields_str) {
4566                 ret = -EINVAL;
4567                 goto out;
4568         }
4569         ret = 0;
4570  out:
4571         return ret;
4572 }
4573
4574 static int create_var_fields(struct hist_trigger_data *hist_data,
4575                              struct trace_event_file *file)
4576 {
4577         unsigned int i, j = hist_data->n_vals;
4578         int ret = 0;
4579
4580         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4581
4582         for (i = 0; i < n_vars; i++) {
4583                 char *var_name = hist_data->attrs->var_defs.name[i];
4584                 char *expr = hist_data->attrs->var_defs.expr[i];
4585
4586                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4587                 if (ret)
4588                         goto out;
4589         }
4590  out:
4591         return ret;
4592 }
4593
4594 static void free_var_defs(struct hist_trigger_data *hist_data)
4595 {
4596         unsigned int i;
4597
4598         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4599                 kfree(hist_data->attrs->var_defs.name[i]);
4600                 kfree(hist_data->attrs->var_defs.expr[i]);
4601         }
4602
4603         hist_data->attrs->var_defs.n_vars = 0;
4604 }
4605
4606 static int parse_var_defs(struct hist_trigger_data *hist_data)
4607 {
4608         struct trace_array *tr = hist_data->event_file->tr;
4609         char *s, *str, *var_name, *field_str;
4610         unsigned int i, j, n_vars = 0;
4611         int ret = 0;
4612
4613         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4614                 str = hist_data->attrs->assignment_str[i];
4615                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4616                         field_str = strsep(&str, ",");
4617                         if (!field_str)
4618                                 break;
4619
4620                         var_name = strsep(&field_str, "=");
4621                         if (!var_name || !field_str) {
4622                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4623                                          errpos(var_name));
4624                                 ret = -EINVAL;
4625                                 goto free;
4626                         }
4627
4628                         if (n_vars == TRACING_MAP_VARS_MAX) {
4629                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4630                                 ret = -EINVAL;
4631                                 goto free;
4632                         }
4633
4634                         s = kstrdup(var_name, GFP_KERNEL);
4635                         if (!s) {
4636                                 ret = -ENOMEM;
4637                                 goto free;
4638                         }
4639                         hist_data->attrs->var_defs.name[n_vars] = s;
4640
4641                         s = kstrdup(field_str, GFP_KERNEL);
4642                         if (!s) {
4643                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4644                                 ret = -ENOMEM;
4645                                 goto free;
4646                         }
4647                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4648
4649                         hist_data->attrs->var_defs.n_vars = n_vars;
4650                 }
4651         }
4652
4653         return ret;
4654  free:
4655         free_var_defs(hist_data);
4656
4657         return ret;
4658 }
4659
4660 static int create_hist_fields(struct hist_trigger_data *hist_data,
4661                               struct trace_event_file *file)
4662 {
4663         int ret;
4664
4665         ret = parse_var_defs(hist_data);
4666         if (ret)
4667                 goto out;
4668
4669         ret = create_val_fields(hist_data, file);
4670         if (ret)
4671                 goto out;
4672
4673         ret = create_var_fields(hist_data, file);
4674         if (ret)
4675                 goto out;
4676
4677         ret = create_key_fields(hist_data, file);
4678         if (ret)
4679                 goto out;
4680  out:
4681         free_var_defs(hist_data);
4682
4683         return ret;
4684 }
4685
4686 static int is_descending(const char *str)
4687 {
4688         if (!str)
4689                 return 0;
4690
4691         if (strcmp(str, "descending") == 0)
4692                 return 1;
4693
4694         if (strcmp(str, "ascending") == 0)
4695                 return 0;
4696
4697         return -EINVAL;
4698 }
4699
4700 static int create_sort_keys(struct hist_trigger_data *hist_data)
4701 {
4702         char *fields_str = hist_data->attrs->sort_key_str;
4703         struct tracing_map_sort_key *sort_key;
4704         int descending, ret = 0;
4705         unsigned int i, j, k;
4706
4707         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4708
4709         if (!fields_str)
4710                 goto out;
4711
4712         strsep(&fields_str, "=");
4713         if (!fields_str) {
4714                 ret = -EINVAL;
4715                 goto out;
4716         }
4717
4718         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4719                 struct hist_field *hist_field;
4720                 char *field_str, *field_name;
4721                 const char *test_name;
4722
4723                 sort_key = &hist_data->sort_keys[i];
4724
4725                 field_str = strsep(&fields_str, ",");
4726                 if (!field_str) {
4727                         if (i == 0)
4728                                 ret = -EINVAL;
4729                         break;
4730                 }
4731
4732                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4733                         ret = -EINVAL;
4734                         break;
4735                 }
4736
4737                 field_name = strsep(&field_str, ".");
4738                 if (!field_name) {
4739                         ret = -EINVAL;
4740                         break;
4741                 }
4742
4743                 if (strcmp(field_name, "hitcount") == 0) {
4744                         descending = is_descending(field_str);
4745                         if (descending < 0) {
4746                                 ret = descending;
4747                                 break;
4748                         }
4749                         sort_key->descending = descending;
4750                         continue;
4751                 }
4752
4753                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4754                         unsigned int idx;
4755
4756                         hist_field = hist_data->fields[j];
4757                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4758                                 continue;
4759
4760                         idx = k++;
4761
4762                         test_name = hist_field_name(hist_field, 0);
4763
4764                         if (strcmp(field_name, test_name) == 0) {
4765                                 sort_key->field_idx = idx;
4766                                 descending = is_descending(field_str);
4767                                 if (descending < 0) {
4768                                         ret = descending;
4769                                         goto out;
4770                                 }
4771                                 sort_key->descending = descending;
4772                                 break;
4773                         }
4774                 }
4775                 if (j == hist_data->n_fields) {
4776                         ret = -EINVAL;
4777                         break;
4778                 }
4779         }
4780
4781         hist_data->n_sort_keys = i;
4782  out:
4783         return ret;
4784 }
4785
4786 static void destroy_actions(struct hist_trigger_data *hist_data)
4787 {
4788         unsigned int i;
4789
4790         for (i = 0; i < hist_data->n_actions; i++) {
4791                 struct action_data *data = hist_data->actions[i];
4792
4793                 if (data->handler == HANDLER_ONMATCH)
4794                         onmatch_destroy(data);
4795                 else if (data->handler == HANDLER_ONMAX ||
4796                          data->handler == HANDLER_ONCHANGE)
4797                         track_data_destroy(hist_data, data);
4798                 else
4799                         kfree(data);
4800         }
4801 }
4802
4803 static int parse_actions(struct hist_trigger_data *hist_data)
4804 {
4805         struct trace_array *tr = hist_data->event_file->tr;
4806         struct action_data *data;
4807         unsigned int i;
4808         int ret = 0;
4809         char *str;
4810         int len;
4811
4812         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4813                 str = hist_data->attrs->action_str[i];
4814
4815                 if ((len = str_has_prefix(str, "onmatch("))) {
4816                         char *action_str = str + len;
4817
4818                         data = onmatch_parse(tr, action_str);
4819                         if (IS_ERR(data)) {
4820                                 ret = PTR_ERR(data);
4821                                 break;
4822                         }
4823                 } else if ((len = str_has_prefix(str, "onmax("))) {
4824                         char *action_str = str + len;
4825
4826                         data = track_data_parse(hist_data, action_str,
4827                                                 HANDLER_ONMAX);
4828                         if (IS_ERR(data)) {
4829                                 ret = PTR_ERR(data);
4830                                 break;
4831                         }
4832                 } else if ((len = str_has_prefix(str, "onchange("))) {
4833                         char *action_str = str + len;
4834
4835                         data = track_data_parse(hist_data, action_str,
4836                                                 HANDLER_ONCHANGE);
4837                         if (IS_ERR(data)) {
4838                                 ret = PTR_ERR(data);
4839                                 break;
4840                         }
4841                 } else {
4842                         ret = -EINVAL;
4843                         break;
4844                 }
4845
4846                 hist_data->actions[hist_data->n_actions++] = data;
4847         }
4848
4849         return ret;
4850 }
4851
4852 static int create_actions(struct hist_trigger_data *hist_data)
4853 {
4854         struct action_data *data;
4855         unsigned int i;
4856         int ret = 0;
4857
4858         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4859                 data = hist_data->actions[i];
4860
4861                 if (data->handler == HANDLER_ONMATCH) {
4862                         ret = onmatch_create(hist_data, data);
4863                         if (ret)
4864                                 break;
4865                 } else if (data->handler == HANDLER_ONMAX ||
4866                            data->handler == HANDLER_ONCHANGE) {
4867                         ret = track_data_create(hist_data, data);
4868                         if (ret)
4869                                 break;
4870                 } else {
4871                         ret = -EINVAL;
4872                         break;
4873                 }
4874         }
4875
4876         return ret;
4877 }
4878
4879 static void print_actions(struct seq_file *m,
4880                           struct hist_trigger_data *hist_data,
4881                           struct tracing_map_elt *elt)
4882 {
4883         unsigned int i;
4884
4885         for (i = 0; i < hist_data->n_actions; i++) {
4886                 struct action_data *data = hist_data->actions[i];
4887
4888                 if (data->action == ACTION_SNAPSHOT)
4889                         continue;
4890
4891                 if (data->handler == HANDLER_ONMAX ||
4892                     data->handler == HANDLER_ONCHANGE)
4893                         track_data_print(m, hist_data, elt, data);
4894         }
4895 }
4896
4897 static void print_action_spec(struct seq_file *m,
4898                               struct hist_trigger_data *hist_data,
4899                               struct action_data *data)
4900 {
4901         unsigned int i;
4902
4903         if (data->action == ACTION_SAVE) {
4904                 for (i = 0; i < hist_data->n_save_vars; i++) {
4905                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4906                         if (i < hist_data->n_save_vars - 1)
4907                                 seq_puts(m, ",");
4908                 }
4909         } else if (data->action == ACTION_TRACE) {
4910                 if (data->use_trace_keyword)
4911                         seq_printf(m, "%s", data->synth_event_name);
4912                 for (i = 0; i < data->n_params; i++) {
4913                         if (i || data->use_trace_keyword)
4914                                 seq_puts(m, ",");
4915                         seq_printf(m, "%s", data->params[i]);
4916                 }
4917         }
4918 }
4919
4920 static void print_track_data_spec(struct seq_file *m,
4921                                   struct hist_trigger_data *hist_data,
4922                                   struct action_data *data)
4923 {
4924         if (data->handler == HANDLER_ONMAX)
4925                 seq_puts(m, ":onmax(");
4926         else if (data->handler == HANDLER_ONCHANGE)
4927                 seq_puts(m, ":onchange(");
4928         seq_printf(m, "%s", data->track_data.var_str);
4929         seq_printf(m, ").%s(", data->action_name);
4930
4931         print_action_spec(m, hist_data, data);
4932
4933         seq_puts(m, ")");
4934 }
4935
4936 static void print_onmatch_spec(struct seq_file *m,
4937                                struct hist_trigger_data *hist_data,
4938                                struct action_data *data)
4939 {
4940         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4941                    data->match_data.event);
4942
4943         seq_printf(m, "%s(", data->action_name);
4944
4945         print_action_spec(m, hist_data, data);
4946
4947         seq_puts(m, ")");
4948 }
4949
4950 static bool actions_match(struct hist_trigger_data *hist_data,
4951                           struct hist_trigger_data *hist_data_test)
4952 {
4953         unsigned int i, j;
4954
4955         if (hist_data->n_actions != hist_data_test->n_actions)
4956                 return false;
4957
4958         for (i = 0; i < hist_data->n_actions; i++) {
4959                 struct action_data *data = hist_data->actions[i];
4960                 struct action_data *data_test = hist_data_test->actions[i];
4961                 char *action_name, *action_name_test;
4962
4963                 if (data->handler != data_test->handler)
4964                         return false;
4965                 if (data->action != data_test->action)
4966                         return false;
4967
4968                 if (data->n_params != data_test->n_params)
4969                         return false;
4970
4971                 for (j = 0; j < data->n_params; j++) {
4972                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4973                                 return false;
4974                 }
4975
4976                 if (data->use_trace_keyword)
4977                         action_name = data->synth_event_name;
4978                 else
4979                         action_name = data->action_name;
4980
4981                 if (data_test->use_trace_keyword)
4982                         action_name_test = data_test->synth_event_name;
4983                 else
4984                         action_name_test = data_test->action_name;
4985
4986                 if (strcmp(action_name, action_name_test) != 0)
4987                         return false;
4988
4989                 if (data->handler == HANDLER_ONMATCH) {
4990                         if (strcmp(data->match_data.event_system,
4991                                    data_test->match_data.event_system) != 0)
4992                                 return false;
4993                         if (strcmp(data->match_data.event,
4994                                    data_test->match_data.event) != 0)
4995                                 return false;
4996                 } else if (data->handler == HANDLER_ONMAX ||
4997                            data->handler == HANDLER_ONCHANGE) {
4998                         if (strcmp(data->track_data.var_str,
4999                                    data_test->track_data.var_str) != 0)
5000                                 return false;
5001                 }
5002         }
5003
5004         return true;
5005 }
5006
5007
5008 static void print_actions_spec(struct seq_file *m,
5009                                struct hist_trigger_data *hist_data)
5010 {
5011         unsigned int i;
5012
5013         for (i = 0; i < hist_data->n_actions; i++) {
5014                 struct action_data *data = hist_data->actions[i];
5015
5016                 if (data->handler == HANDLER_ONMATCH)
5017                         print_onmatch_spec(m, hist_data, data);
5018                 else if (data->handler == HANDLER_ONMAX ||
5019                          data->handler == HANDLER_ONCHANGE)
5020                         print_track_data_spec(m, hist_data, data);
5021         }
5022 }
5023
5024 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5025 {
5026         unsigned int i;
5027
5028         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5029                 kfree(hist_data->field_var_hists[i]->cmd);
5030                 kfree(hist_data->field_var_hists[i]);
5031         }
5032 }
5033
5034 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5035 {
5036         if (!hist_data)
5037                 return;
5038
5039         destroy_hist_trigger_attrs(hist_data->attrs);
5040         destroy_hist_fields(hist_data);
5041         tracing_map_destroy(hist_data->map);
5042
5043         destroy_actions(hist_data);
5044         destroy_field_vars(hist_data);
5045         destroy_field_var_hists(hist_data);
5046
5047         kfree(hist_data);
5048 }
5049
5050 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5051 {
5052         struct tracing_map *map = hist_data->map;
5053         struct ftrace_event_field *field;
5054         struct hist_field *hist_field;
5055         int i, idx = 0;
5056
5057         for_each_hist_field(i, hist_data) {
5058                 hist_field = hist_data->fields[i];
5059                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5060                         tracing_map_cmp_fn_t cmp_fn;
5061
5062                         field = hist_field->field;
5063
5064                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5065                                 cmp_fn = tracing_map_cmp_none;
5066                         else if (!field)
5067                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5068                                                              hist_field->is_signed);
5069                         else if (is_string_field(field))
5070                                 cmp_fn = tracing_map_cmp_string;
5071                         else
5072                                 cmp_fn = tracing_map_cmp_num(field->size,
5073                                                              field->is_signed);
5074                         idx = tracing_map_add_key_field(map,
5075                                                         hist_field->offset,
5076                                                         cmp_fn);
5077                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5078                         idx = tracing_map_add_sum_field(map);
5079
5080                 if (idx < 0)
5081                         return idx;
5082
5083                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5084                         idx = tracing_map_add_var(map);
5085                         if (idx < 0)
5086                                 return idx;
5087                         hist_field->var.idx = idx;
5088                         hist_field->var.hist_data = hist_data;
5089                 }
5090         }
5091
5092         return 0;
5093 }
5094
5095 static struct hist_trigger_data *
5096 create_hist_data(unsigned int map_bits,
5097                  struct hist_trigger_attrs *attrs,
5098                  struct trace_event_file *file,
5099                  bool remove)
5100 {
5101         const struct tracing_map_ops *map_ops = NULL;
5102         struct hist_trigger_data *hist_data;
5103         int ret = 0;
5104
5105         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5106         if (!hist_data)
5107                 return ERR_PTR(-ENOMEM);
5108
5109         hist_data->attrs = attrs;
5110         hist_data->remove = remove;
5111         hist_data->event_file = file;
5112
5113         ret = parse_actions(hist_data);
5114         if (ret)
5115                 goto free;
5116
5117         ret = create_hist_fields(hist_data, file);
5118         if (ret)
5119                 goto free;
5120
5121         ret = create_sort_keys(hist_data);
5122         if (ret)
5123                 goto free;
5124
5125         map_ops = &hist_trigger_elt_data_ops;
5126
5127         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5128                                             map_ops, hist_data);
5129         if (IS_ERR(hist_data->map)) {
5130                 ret = PTR_ERR(hist_data->map);
5131                 hist_data->map = NULL;
5132                 goto free;
5133         }
5134
5135         ret = create_tracing_map_fields(hist_data);
5136         if (ret)
5137                 goto free;
5138  out:
5139         return hist_data;
5140  free:
5141         hist_data->attrs = NULL;
5142
5143         destroy_hist_data(hist_data);
5144
5145         hist_data = ERR_PTR(ret);
5146
5147         goto out;
5148 }
5149
5150 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5151                                     struct tracing_map_elt *elt, void *rec,
5152                                     struct ring_buffer_event *rbe,
5153                                     u64 *var_ref_vals)
5154 {
5155         struct hist_elt_data *elt_data;
5156         struct hist_field *hist_field;
5157         unsigned int i, var_idx;
5158         u64 hist_val;
5159
5160         elt_data = elt->private_data;
5161         elt_data->var_ref_vals = var_ref_vals;
5162
5163         for_each_hist_val_field(i, hist_data) {
5164                 hist_field = hist_data->fields[i];
5165                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5166                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5167                         var_idx = hist_field->var.idx;
5168                         tracing_map_set_var(elt, var_idx, hist_val);
5169                         continue;
5170                 }
5171                 tracing_map_update_sum(elt, i, hist_val);
5172         }
5173
5174         for_each_hist_key_field(i, hist_data) {
5175                 hist_field = hist_data->fields[i];
5176                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5177                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5178                         var_idx = hist_field->var.idx;
5179                         tracing_map_set_var(elt, var_idx, hist_val);
5180                 }
5181         }
5182
5183         update_field_vars(hist_data, elt, rbe, rec);
5184 }
5185
5186 static inline void add_to_key(char *compound_key, void *key,
5187                               struct hist_field *key_field, void *rec)
5188 {
5189         size_t size = key_field->size;
5190
5191         if (key_field->flags & HIST_FIELD_FL_STRING) {
5192                 struct ftrace_event_field *field;
5193
5194                 field = key_field->field;
5195                 if (field->filter_type == FILTER_DYN_STRING)
5196                         size = *(u32 *)(rec + field->offset) >> 16;
5197                 else if (field->filter_type == FILTER_PTR_STRING)
5198                         size = strlen(key);
5199                 else if (field->filter_type == FILTER_STATIC_STRING)
5200                         size = field->size;
5201
5202                 /* ensure NULL-termination */
5203                 if (size > key_field->size - 1)
5204                         size = key_field->size - 1;
5205
5206                 strncpy(compound_key + key_field->offset, (char *)key, size);
5207         } else
5208                 memcpy(compound_key + key_field->offset, key, size);
5209 }
5210
5211 static void
5212 hist_trigger_actions(struct hist_trigger_data *hist_data,
5213                      struct tracing_map_elt *elt, void *rec,
5214                      struct ring_buffer_event *rbe, void *key,
5215                      u64 *var_ref_vals)
5216 {
5217         struct action_data *data;
5218         unsigned int i;
5219
5220         for (i = 0; i < hist_data->n_actions; i++) {
5221                 data = hist_data->actions[i];
5222                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
5223         }
5224 }
5225
5226 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
5227                                struct ring_buffer_event *rbe)
5228 {
5229         struct hist_trigger_data *hist_data = data->private_data;
5230         bool use_compound_key = (hist_data->n_keys > 1);
5231         unsigned long entries[HIST_STACKTRACE_DEPTH];
5232         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5233         char compound_key[HIST_KEY_SIZE_MAX];
5234         struct tracing_map_elt *elt = NULL;
5235         struct hist_field *key_field;
5236         u64 field_contents;
5237         void *key = NULL;
5238         unsigned int i;
5239
5240         memset(compound_key, 0, hist_data->key_size);
5241
5242         for_each_hist_key_field(i, hist_data) {
5243                 key_field = hist_data->fields[i];
5244
5245                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5246                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5247                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5248                                          HIST_STACKTRACE_SKIP);
5249                         key = entries;
5250                 } else {
5251                         field_contents = key_field->fn(key_field, elt, rbe, rec);
5252                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5253                                 key = (void *)(unsigned long)field_contents;
5254                                 use_compound_key = true;
5255                         } else
5256                                 key = (void *)&field_contents;
5257                 }
5258
5259                 if (use_compound_key)
5260                         add_to_key(compound_key, key, key_field, rec);
5261         }
5262
5263         if (use_compound_key)
5264                 key = compound_key;
5265
5266         if (hist_data->n_var_refs &&
5267             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5268                 return;
5269
5270         elt = tracing_map_insert(hist_data->map, key);
5271         if (!elt)
5272                 return;
5273
5274         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
5275
5276         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5277                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
5278 }
5279
5280 static void hist_trigger_stacktrace_print(struct seq_file *m,
5281                                           unsigned long *stacktrace_entries,
5282                                           unsigned int max_entries)
5283 {
5284         char str[KSYM_SYMBOL_LEN];
5285         unsigned int spaces = 8;
5286         unsigned int i;
5287
5288         for (i = 0; i < max_entries; i++) {
5289                 if (!stacktrace_entries[i])
5290                         return;
5291
5292                 seq_printf(m, "%*c", 1 + spaces, ' ');
5293                 sprint_symbol(str, stacktrace_entries[i]);
5294                 seq_printf(m, "%s\n", str);
5295         }
5296 }
5297
5298 static void hist_trigger_print_key(struct seq_file *m,
5299                                    struct hist_trigger_data *hist_data,
5300                                    void *key,
5301                                    struct tracing_map_elt *elt)
5302 {
5303         struct hist_field *key_field;
5304         char str[KSYM_SYMBOL_LEN];
5305         bool multiline = false;
5306         const char *field_name;
5307         unsigned int i;
5308         u64 uval;
5309
5310         seq_puts(m, "{ ");
5311
5312         for_each_hist_key_field(i, hist_data) {
5313                 key_field = hist_data->fields[i];
5314
5315                 if (i > hist_data->n_vals)
5316                         seq_puts(m, ", ");
5317
5318                 field_name = hist_field_name(key_field, 0);
5319
5320                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5321                         uval = *(u64 *)(key + key_field->offset);
5322                         seq_printf(m, "%s: %llx", field_name, uval);
5323                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5324                         uval = *(u64 *)(key + key_field->offset);
5325                         sprint_symbol_no_offset(str, uval);
5326                         seq_printf(m, "%s: [%llx] %-45s", field_name,
5327                                    uval, str);
5328                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5329                         uval = *(u64 *)(key + key_field->offset);
5330                         sprint_symbol(str, uval);
5331                         seq_printf(m, "%s: [%llx] %-55s", field_name,
5332                                    uval, str);
5333                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5334                         struct hist_elt_data *elt_data = elt->private_data;
5335                         char *comm;
5336
5337                         if (WARN_ON_ONCE(!elt_data))
5338                                 return;
5339
5340                         comm = elt_data->comm;
5341
5342                         uval = *(u64 *)(key + key_field->offset);
5343                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5344                                    comm, uval);
5345                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5346                         const char *syscall_name;
5347
5348                         uval = *(u64 *)(key + key_field->offset);
5349                         syscall_name = get_syscall_name(uval);
5350                         if (!syscall_name)
5351                                 syscall_name = "unknown_syscall";
5352
5353                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5354                                    syscall_name, uval);
5355                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5356                         seq_puts(m, "stacktrace:\n");
5357                         hist_trigger_stacktrace_print(m,
5358                                                       key + key_field->offset,
5359                                                       HIST_STACKTRACE_DEPTH);
5360                         multiline = true;
5361                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5362                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5363                                    *(u64 *)(key + key_field->offset));
5364                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5365                         seq_printf(m, "%s: %-50s", field_name,
5366                                    (char *)(key + key_field->offset));
5367                 } else {
5368                         uval = *(u64 *)(key + key_field->offset);
5369                         seq_printf(m, "%s: %10llu", field_name, uval);
5370                 }
5371         }
5372
5373         if (!multiline)
5374                 seq_puts(m, " ");
5375
5376         seq_puts(m, "}");
5377 }
5378
5379 static void hist_trigger_entry_print(struct seq_file *m,
5380                                      struct hist_trigger_data *hist_data,
5381                                      void *key,
5382                                      struct tracing_map_elt *elt)
5383 {
5384         const char *field_name;
5385         unsigned int i;
5386
5387         hist_trigger_print_key(m, hist_data, key, elt);
5388
5389         seq_printf(m, " hitcount: %10llu",
5390                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5391
5392         for (i = 1; i < hist_data->n_vals; i++) {
5393                 field_name = hist_field_name(hist_data->fields[i], 0);
5394
5395                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5396                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5397                         continue;
5398
5399                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5400                         seq_printf(m, "  %s: %10llx", field_name,
5401                                    tracing_map_read_sum(elt, i));
5402                 } else {
5403                         seq_printf(m, "  %s: %10llu", field_name,
5404                                    tracing_map_read_sum(elt, i));
5405                 }
5406         }
5407
5408         print_actions(m, hist_data, elt);
5409
5410         seq_puts(m, "\n");
5411 }
5412
5413 static int print_entries(struct seq_file *m,
5414                          struct hist_trigger_data *hist_data)
5415 {
5416         struct tracing_map_sort_entry **sort_entries = NULL;
5417         struct tracing_map *map = hist_data->map;
5418         int i, n_entries;
5419
5420         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5421                                              hist_data->n_sort_keys,
5422                                              &sort_entries);
5423         if (n_entries < 0)
5424                 return n_entries;
5425
5426         for (i = 0; i < n_entries; i++)
5427                 hist_trigger_entry_print(m, hist_data,
5428                                          sort_entries[i]->key,
5429                                          sort_entries[i]->elt);
5430
5431         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5432
5433         return n_entries;
5434 }
5435
5436 static void hist_trigger_show(struct seq_file *m,
5437                               struct event_trigger_data *data, int n)
5438 {
5439         struct hist_trigger_data *hist_data;
5440         int n_entries;
5441
5442         if (n > 0)
5443                 seq_puts(m, "\n\n");
5444
5445         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5446         data->ops->print(m, data->ops, data);
5447         seq_puts(m, "#\n\n");
5448
5449         hist_data = data->private_data;
5450         n_entries = print_entries(m, hist_data);
5451         if (n_entries < 0)
5452                 n_entries = 0;
5453
5454         track_data_snapshot_print(m, hist_data);
5455
5456         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5457                    (u64)atomic64_read(&hist_data->map->hits),
5458                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5459 }
5460
5461 static int hist_show(struct seq_file *m, void *v)
5462 {
5463         struct event_trigger_data *data;
5464         struct trace_event_file *event_file;
5465         int n = 0, ret = 0;
5466
5467         mutex_lock(&event_mutex);
5468
5469         event_file = event_file_data(m->private);
5470         if (unlikely(!event_file)) {
5471                 ret = -ENODEV;
5472                 goto out_unlock;
5473         }
5474
5475         list_for_each_entry_rcu(data, &event_file->triggers, list) {
5476                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5477                         hist_trigger_show(m, data, n++);
5478         }
5479
5480  out_unlock:
5481         mutex_unlock(&event_mutex);
5482
5483         return ret;
5484 }
5485
5486 static int event_hist_open(struct inode *inode, struct file *file)
5487 {
5488         return single_open(file, hist_show, file);
5489 }
5490
5491 const struct file_operations event_hist_fops = {
5492         .open = event_hist_open,
5493         .read = seq_read,
5494         .llseek = seq_lseek,
5495         .release = single_release,
5496 };
5497
5498 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5499 {
5500         const char *field_name = hist_field_name(hist_field, 0);
5501
5502         if (hist_field->var.name)
5503                 seq_printf(m, "%s=", hist_field->var.name);
5504
5505         if (hist_field->flags & HIST_FIELD_FL_CPU)
5506                 seq_puts(m, "cpu");
5507         else if (field_name) {
5508                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5509                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5510                         seq_putc(m, '$');
5511                 seq_printf(m, "%s", field_name);
5512         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5513                 seq_puts(m, "common_timestamp");
5514
5515         if (hist_field->flags) {
5516                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5517                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5518                         const char *flags = get_hist_field_flags(hist_field);
5519
5520                         if (flags)
5521                                 seq_printf(m, ".%s", flags);
5522                 }
5523         }
5524 }
5525
5526 static int event_hist_trigger_print(struct seq_file *m,
5527                                     struct event_trigger_ops *ops,
5528                                     struct event_trigger_data *data)
5529 {
5530         struct hist_trigger_data *hist_data = data->private_data;
5531         struct hist_field *field;
5532         bool have_var = false;
5533         unsigned int i;
5534
5535         seq_puts(m, "hist:");
5536
5537         if (data->name)
5538                 seq_printf(m, "%s:", data->name);
5539
5540         seq_puts(m, "keys=");
5541
5542         for_each_hist_key_field(i, hist_data) {
5543                 field = hist_data->fields[i];
5544
5545                 if (i > hist_data->n_vals)
5546                         seq_puts(m, ",");
5547
5548                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5549                         seq_puts(m, "stacktrace");
5550                 else
5551                         hist_field_print(m, field);
5552         }
5553
5554         seq_puts(m, ":vals=");
5555
5556         for_each_hist_val_field(i, hist_data) {
5557                 field = hist_data->fields[i];
5558                 if (field->flags & HIST_FIELD_FL_VAR) {
5559                         have_var = true;
5560                         continue;
5561                 }
5562
5563                 if (i == HITCOUNT_IDX)
5564                         seq_puts(m, "hitcount");
5565                 else {
5566                         seq_puts(m, ",");
5567                         hist_field_print(m, field);
5568                 }
5569         }
5570
5571         if (have_var) {
5572                 unsigned int n = 0;
5573
5574                 seq_puts(m, ":");
5575
5576                 for_each_hist_val_field(i, hist_data) {
5577                         field = hist_data->fields[i];
5578
5579                         if (field->flags & HIST_FIELD_FL_VAR) {
5580                                 if (n++)
5581                                         seq_puts(m, ",");
5582                                 hist_field_print(m, field);
5583                         }
5584                 }
5585         }
5586
5587         seq_puts(m, ":sort=");
5588
5589         for (i = 0; i < hist_data->n_sort_keys; i++) {
5590                 struct tracing_map_sort_key *sort_key;
5591                 unsigned int idx, first_key_idx;
5592
5593                 /* skip VAR vals */
5594                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5595
5596                 sort_key = &hist_data->sort_keys[i];
5597                 idx = sort_key->field_idx;
5598
5599                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5600                         return -EINVAL;
5601
5602                 if (i > 0)
5603                         seq_puts(m, ",");
5604
5605                 if (idx == HITCOUNT_IDX)
5606                         seq_puts(m, "hitcount");
5607                 else {
5608                         if (idx >= first_key_idx)
5609                                 idx += hist_data->n_vars;
5610                         hist_field_print(m, hist_data->fields[idx]);
5611                 }
5612
5613                 if (sort_key->descending)
5614                         seq_puts(m, ".descending");
5615         }
5616         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5617         if (hist_data->enable_timestamps)
5618                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5619
5620         print_actions_spec(m, hist_data);
5621
5622         if (data->filter_str)
5623                 seq_printf(m, " if %s", data->filter_str);
5624
5625         if (data->paused)
5626                 seq_puts(m, " [paused]");
5627         else
5628                 seq_puts(m, " [active]");
5629
5630         seq_putc(m, '\n');
5631
5632         return 0;
5633 }
5634
5635 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5636                                    struct event_trigger_data *data)
5637 {
5638         struct hist_trigger_data *hist_data = data->private_data;
5639
5640         if (!data->ref && hist_data->attrs->name)
5641                 save_named_trigger(hist_data->attrs->name, data);
5642
5643         data->ref++;
5644
5645         return 0;
5646 }
5647
5648 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5649 {
5650         struct trace_event_file *file;
5651         unsigned int i;
5652         char *cmd;
5653         int ret;
5654
5655         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5656                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5657                 cmd = hist_data->field_var_hists[i]->cmd;
5658                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5659                                               "!hist", "hist", cmd);
5660         }
5661 }
5662
5663 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5664                                     struct event_trigger_data *data)
5665 {
5666         struct hist_trigger_data *hist_data = data->private_data;
5667
5668         if (WARN_ON_ONCE(data->ref <= 0))
5669                 return;
5670
5671         data->ref--;
5672         if (!data->ref) {
5673                 if (data->name)
5674                         del_named_trigger(data);
5675
5676                 trigger_data_free(data);
5677
5678                 remove_hist_vars(hist_data);
5679
5680                 unregister_field_var_hists(hist_data);
5681
5682                 destroy_hist_data(hist_data);
5683         }
5684 }
5685
5686 static struct event_trigger_ops event_hist_trigger_ops = {
5687         .func                   = event_hist_trigger,
5688         .print                  = event_hist_trigger_print,
5689         .init                   = event_hist_trigger_init,
5690         .free                   = event_hist_trigger_free,
5691 };
5692
5693 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5694                                          struct event_trigger_data *data)
5695 {
5696         data->ref++;
5697
5698         save_named_trigger(data->named_data->name, data);
5699
5700         event_hist_trigger_init(ops, data->named_data);
5701
5702         return 0;
5703 }
5704
5705 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5706                                           struct event_trigger_data *data)
5707 {
5708         if (WARN_ON_ONCE(data->ref <= 0))
5709                 return;
5710
5711         event_hist_trigger_free(ops, data->named_data);
5712
5713         data->ref--;
5714         if (!data->ref) {
5715                 del_named_trigger(data);
5716                 trigger_data_free(data);
5717         }
5718 }
5719
5720 static struct event_trigger_ops event_hist_trigger_named_ops = {
5721         .func                   = event_hist_trigger,
5722         .print                  = event_hist_trigger_print,
5723         .init                   = event_hist_trigger_named_init,
5724         .free                   = event_hist_trigger_named_free,
5725 };
5726
5727 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5728                                                             char *param)
5729 {
5730         return &event_hist_trigger_ops;
5731 }
5732
5733 static void hist_clear(struct event_trigger_data *data)
5734 {
5735         struct hist_trigger_data *hist_data = data->private_data;
5736
5737         if (data->name)
5738                 pause_named_trigger(data);
5739
5740         tracepoint_synchronize_unregister();
5741
5742         tracing_map_clear(hist_data->map);
5743
5744         if (data->name)
5745                 unpause_named_trigger(data);
5746 }
5747
5748 static bool compatible_field(struct ftrace_event_field *field,
5749                              struct ftrace_event_field *test_field)
5750 {
5751         if (field == test_field)
5752                 return true;
5753         if (field == NULL || test_field == NULL)
5754                 return false;
5755         if (strcmp(field->name, test_field->name) != 0)
5756                 return false;
5757         if (strcmp(field->type, test_field->type) != 0)
5758                 return false;
5759         if (field->size != test_field->size)
5760                 return false;
5761         if (field->is_signed != test_field->is_signed)
5762                 return false;
5763
5764         return true;
5765 }
5766
5767 static bool hist_trigger_match(struct event_trigger_data *data,
5768                                struct event_trigger_data *data_test,
5769                                struct event_trigger_data *named_data,
5770                                bool ignore_filter)
5771 {
5772         struct tracing_map_sort_key *sort_key, *sort_key_test;
5773         struct hist_trigger_data *hist_data, *hist_data_test;
5774         struct hist_field *key_field, *key_field_test;
5775         unsigned int i;
5776
5777         if (named_data && (named_data != data_test) &&
5778             (named_data != data_test->named_data))
5779                 return false;
5780
5781         if (!named_data && is_named_trigger(data_test))
5782                 return false;
5783
5784         hist_data = data->private_data;
5785         hist_data_test = data_test->private_data;
5786
5787         if (hist_data->n_vals != hist_data_test->n_vals ||
5788             hist_data->n_fields != hist_data_test->n_fields ||
5789             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5790                 return false;
5791
5792         if (!ignore_filter) {
5793                 if ((data->filter_str && !data_test->filter_str) ||
5794                    (!data->filter_str && data_test->filter_str))
5795                         return false;
5796         }
5797
5798         for_each_hist_field(i, hist_data) {
5799                 key_field = hist_data->fields[i];
5800                 key_field_test = hist_data_test->fields[i];
5801
5802                 if (key_field->flags != key_field_test->flags)
5803                         return false;
5804                 if (!compatible_field(key_field->field, key_field_test->field))
5805                         return false;
5806                 if (key_field->offset != key_field_test->offset)
5807                         return false;
5808                 if (key_field->size != key_field_test->size)
5809                         return false;
5810                 if (key_field->is_signed != key_field_test->is_signed)
5811                         return false;
5812                 if (!!key_field->var.name != !!key_field_test->var.name)
5813                         return false;
5814                 if (key_field->var.name &&
5815                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5816                         return false;
5817         }
5818
5819         for (i = 0; i < hist_data->n_sort_keys; i++) {
5820                 sort_key = &hist_data->sort_keys[i];
5821                 sort_key_test = &hist_data_test->sort_keys[i];
5822
5823                 if (sort_key->field_idx != sort_key_test->field_idx ||
5824                     sort_key->descending != sort_key_test->descending)
5825                         return false;
5826         }
5827
5828         if (!ignore_filter && data->filter_str &&
5829             (strcmp(data->filter_str, data_test->filter_str) != 0))
5830                 return false;
5831
5832         if (!actions_match(hist_data, hist_data_test))
5833                 return false;
5834
5835         return true;
5836 }
5837
5838 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5839                                  struct event_trigger_data *data,
5840                                  struct trace_event_file *file)
5841 {
5842         struct hist_trigger_data *hist_data = data->private_data;
5843         struct event_trigger_data *test, *named_data = NULL;
5844         struct trace_array *tr = file->tr;
5845         int ret = 0;
5846
5847         if (hist_data->attrs->name) {
5848                 named_data = find_named_trigger(hist_data->attrs->name);
5849                 if (named_data) {
5850                         if (!hist_trigger_match(data, named_data, named_data,
5851                                                 true)) {
5852                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5853                                 ret = -EINVAL;
5854                                 goto out;
5855                         }
5856                 }
5857         }
5858
5859         if (hist_data->attrs->name && !named_data)
5860                 goto new;
5861
5862         list_for_each_entry_rcu(test, &file->triggers, list) {
5863                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5864                         if (!hist_trigger_match(data, test, named_data, false))
5865                                 continue;
5866                         if (hist_data->attrs->pause)
5867                                 test->paused = true;
5868                         else if (hist_data->attrs->cont)
5869                                 test->paused = false;
5870                         else if (hist_data->attrs->clear)
5871                                 hist_clear(test);
5872                         else {
5873                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5874                                 ret = -EEXIST;
5875                         }
5876                         goto out;
5877                 }
5878         }
5879  new:
5880         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5881                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5882                 ret = -ENOENT;
5883                 goto out;
5884         }
5885
5886         if (hist_data->attrs->pause)
5887                 data->paused = true;
5888
5889         if (named_data) {
5890                 data->private_data = named_data->private_data;
5891                 set_named_trigger_data(data, named_data);
5892                 data->ops = &event_hist_trigger_named_ops;
5893         }
5894
5895         if (data->ops->init) {
5896                 ret = data->ops->init(data->ops, data);
5897                 if (ret < 0)
5898                         goto out;
5899         }
5900
5901         if (hist_data->enable_timestamps) {
5902                 char *clock = hist_data->attrs->clock;
5903
5904                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5905                 if (ret) {
5906                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5907                         goto out;
5908                 }
5909
5910                 tracing_set_time_stamp_abs(file->tr, true);
5911         }
5912
5913         if (named_data)
5914                 destroy_hist_data(hist_data);
5915
5916         ret++;
5917  out:
5918         return ret;
5919 }
5920
5921 static int hist_trigger_enable(struct event_trigger_data *data,
5922                                struct trace_event_file *file)
5923 {
5924         int ret = 0;
5925
5926         list_add_tail_rcu(&data->list, &file->triggers);
5927
5928         update_cond_flag(file);
5929
5930         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5931                 list_del_rcu(&data->list);
5932                 update_cond_flag(file);
5933                 ret--;
5934         }
5935
5936         return ret;
5937 }
5938
5939 static bool have_hist_trigger_match(struct event_trigger_data *data,
5940                                     struct trace_event_file *file)
5941 {
5942         struct hist_trigger_data *hist_data = data->private_data;
5943         struct event_trigger_data *test, *named_data = NULL;
5944         bool match = false;
5945
5946         if (hist_data->attrs->name)
5947                 named_data = find_named_trigger(hist_data->attrs->name);
5948
5949         list_for_each_entry_rcu(test, &file->triggers, list) {
5950                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5951                         if (hist_trigger_match(data, test, named_data, false)) {
5952                                 match = true;
5953                                 break;
5954                         }
5955                 }
5956         }
5957
5958         return match;
5959 }
5960
5961 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5962                                     struct trace_event_file *file)
5963 {
5964         struct hist_trigger_data *hist_data = data->private_data;
5965         struct event_trigger_data *test, *named_data = NULL;
5966
5967         if (hist_data->attrs->name)
5968                 named_data = find_named_trigger(hist_data->attrs->name);
5969
5970         list_for_each_entry_rcu(test, &file->triggers, list) {
5971                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5972                         if (!hist_trigger_match(data, test, named_data, false))
5973                                 continue;
5974                         hist_data = test->private_data;
5975                         if (check_var_refs(hist_data))
5976                                 return true;
5977                         break;
5978                 }
5979         }
5980
5981         return false;
5982 }
5983
5984 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5985                                     struct event_trigger_data *data,
5986                                     struct trace_event_file *file)
5987 {
5988         struct hist_trigger_data *hist_data = data->private_data;
5989         struct event_trigger_data *test, *named_data = NULL;
5990         bool unregistered = false;
5991
5992         if (hist_data->attrs->name)
5993                 named_data = find_named_trigger(hist_data->attrs->name);
5994
5995         list_for_each_entry_rcu(test, &file->triggers, list) {
5996                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5997                         if (!hist_trigger_match(data, test, named_data, false))
5998                                 continue;
5999                         unregistered = true;
6000                         list_del_rcu(&test->list);
6001                         trace_event_trigger_enable_disable(file, 0);
6002                         update_cond_flag(file);
6003                         break;
6004                 }
6005         }
6006
6007         if (unregistered && test->ops->free)
6008                 test->ops->free(test->ops, test);
6009
6010         if (hist_data->enable_timestamps) {
6011                 if (!hist_data->remove || unregistered)
6012                         tracing_set_time_stamp_abs(file->tr, false);
6013         }
6014 }
6015
6016 static bool hist_file_check_refs(struct trace_event_file *file)
6017 {
6018         struct hist_trigger_data *hist_data;
6019         struct event_trigger_data *test;
6020
6021         list_for_each_entry_rcu(test, &file->triggers, list) {
6022                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6023                         hist_data = test->private_data;
6024                         if (check_var_refs(hist_data))
6025                                 return true;
6026                 }
6027         }
6028
6029         return false;
6030 }
6031
6032 static void hist_unreg_all(struct trace_event_file *file)
6033 {
6034         struct event_trigger_data *test, *n;
6035         struct hist_trigger_data *hist_data;
6036         struct synth_event *se;
6037         const char *se_name;
6038
6039         lockdep_assert_held(&event_mutex);
6040
6041         if (hist_file_check_refs(file))
6042                 return;
6043
6044         list_for_each_entry_safe(test, n, &file->triggers, list) {
6045                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6046                         hist_data = test->private_data;
6047                         list_del_rcu(&test->list);
6048                         trace_event_trigger_enable_disable(file, 0);
6049
6050                         se_name = trace_event_name(file->event_call);
6051                         se = find_synth_event(se_name);
6052                         if (se)
6053                                 se->ref--;
6054
6055                         update_cond_flag(file);
6056                         if (hist_data->enable_timestamps)
6057                                 tracing_set_time_stamp_abs(file->tr, false);
6058                         if (test->ops->free)
6059                                 test->ops->free(test->ops, test);
6060                 }
6061         }
6062 }
6063
6064 static int event_hist_trigger_func(struct event_command *cmd_ops,
6065                                    struct trace_event_file *file,
6066                                    char *glob, char *cmd, char *param)
6067 {
6068         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6069         struct event_trigger_data *trigger_data;
6070         struct hist_trigger_attrs *attrs;
6071         struct event_trigger_ops *trigger_ops;
6072         struct hist_trigger_data *hist_data;
6073         struct synth_event *se;
6074         const char *se_name;
6075         bool remove = false;
6076         char *trigger, *p;
6077         int ret = 0;
6078
6079         lockdep_assert_held(&event_mutex);
6080
6081         if (glob && strlen(glob)) {
6082                 hist_err_clear();
6083                 last_cmd_set(file, param);
6084         }
6085
6086         if (!param)
6087                 return -EINVAL;
6088
6089         if (glob[0] == '!')
6090                 remove = true;
6091
6092         /*
6093          * separate the trigger from the filter (k:v [if filter])
6094          * allowing for whitespace in the trigger
6095          */
6096         p = trigger = param;
6097         do {
6098                 p = strstr(p, "if");
6099                 if (!p)
6100                         break;
6101                 if (p == param)
6102                         return -EINVAL;
6103                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6104                         p++;
6105                         continue;
6106                 }
6107                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6108                         return -EINVAL;
6109                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6110                         p++;
6111                         continue;
6112                 }
6113                 break;
6114         } while (p);
6115
6116         if (!p)
6117                 param = NULL;
6118         else {
6119                 *(p - 1) = '\0';
6120                 param = strstrip(p);
6121                 trigger = strstrip(trigger);
6122         }
6123
6124         attrs = parse_hist_trigger_attrs(file->tr, trigger);
6125         if (IS_ERR(attrs))
6126                 return PTR_ERR(attrs);
6127
6128         if (attrs->map_bits)
6129                 hist_trigger_bits = attrs->map_bits;
6130
6131         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6132         if (IS_ERR(hist_data)) {
6133                 destroy_hist_trigger_attrs(attrs);
6134                 return PTR_ERR(hist_data);
6135         }
6136
6137         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6138
6139         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6140         if (!trigger_data) {
6141                 ret = -ENOMEM;
6142                 goto out_free;
6143         }
6144
6145         trigger_data->count = -1;
6146         trigger_data->ops = trigger_ops;
6147         trigger_data->cmd_ops = cmd_ops;
6148
6149         INIT_LIST_HEAD(&trigger_data->list);
6150         RCU_INIT_POINTER(trigger_data->filter, NULL);
6151
6152         trigger_data->private_data = hist_data;
6153
6154         /* if param is non-empty, it's supposed to be a filter */
6155         if (param && cmd_ops->set_filter) {
6156                 ret = cmd_ops->set_filter(param, trigger_data, file);
6157                 if (ret < 0)
6158                         goto out_free;
6159         }
6160
6161         if (remove) {
6162                 if (!have_hist_trigger_match(trigger_data, file))
6163                         goto out_free;
6164
6165                 if (hist_trigger_check_refs(trigger_data, file)) {
6166                         ret = -EBUSY;
6167                         goto out_free;
6168                 }
6169
6170                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6171                 se_name = trace_event_name(file->event_call);
6172                 se = find_synth_event(se_name);
6173                 if (se)
6174                         se->ref--;
6175                 ret = 0;
6176                 goto out_free;
6177         }
6178
6179         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
6180         /*
6181          * The above returns on success the # of triggers registered,
6182          * but if it didn't register any it returns zero.  Consider no
6183          * triggers registered a failure too.
6184          */
6185         if (!ret) {
6186                 if (!(attrs->pause || attrs->cont || attrs->clear))
6187                         ret = -ENOENT;
6188                 goto out_free;
6189         } else if (ret < 0)
6190                 goto out_free;
6191
6192         if (get_named_trigger_data(trigger_data))
6193                 goto enable;
6194
6195         if (has_hist_vars(hist_data))
6196                 save_hist_vars(hist_data);
6197
6198         ret = create_actions(hist_data);
6199         if (ret)
6200                 goto out_unreg;
6201
6202         ret = tracing_map_init(hist_data->map);
6203         if (ret)
6204                 goto out_unreg;
6205 enable:
6206         ret = hist_trigger_enable(trigger_data, file);
6207         if (ret)
6208                 goto out_unreg;
6209
6210         se_name = trace_event_name(file->event_call);
6211         se = find_synth_event(se_name);
6212         if (se)
6213                 se->ref++;
6214         /* Just return zero, not the number of registered triggers */
6215         ret = 0;
6216  out:
6217         if (ret == 0)
6218                 hist_err_clear();
6219
6220         return ret;
6221  out_unreg:
6222         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6223  out_free:
6224         if (cmd_ops->set_filter)
6225                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6226
6227         remove_hist_vars(hist_data);
6228
6229         kfree(trigger_data);
6230
6231         destroy_hist_data(hist_data);
6232         goto out;
6233 }
6234
6235 static struct event_command trigger_hist_cmd = {
6236         .name                   = "hist",
6237         .trigger_type           = ETT_EVENT_HIST,
6238         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6239         .func                   = event_hist_trigger_func,
6240         .reg                    = hist_register_trigger,
6241         .unreg                  = hist_unregister_trigger,
6242         .unreg_all              = hist_unreg_all,
6243         .get_trigger_ops        = event_hist_get_trigger_ops,
6244         .set_filter             = set_trigger_filter,
6245 };
6246
6247 __init int register_trigger_hist_cmd(void)
6248 {
6249         int ret;
6250
6251         ret = register_event_command(&trigger_hist_cmd);
6252         WARN_ON(ret < 0);
6253
6254         return ret;
6255 }
6256
6257 static void
6258 hist_enable_trigger(struct event_trigger_data *data, void *rec,
6259                     struct ring_buffer_event *event)
6260 {
6261         struct enable_trigger_data *enable_data = data->private_data;
6262         struct event_trigger_data *test;
6263
6264         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
6265                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6266                         if (enable_data->enable)
6267                                 test->paused = false;
6268                         else
6269                                 test->paused = true;
6270                 }
6271         }
6272 }
6273
6274 static void
6275 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
6276                           struct ring_buffer_event *event)
6277 {
6278         if (!data->count)
6279                 return;
6280
6281         if (data->count != -1)
6282                 (data->count)--;
6283
6284         hist_enable_trigger(data, rec, event);
6285 }
6286
6287 static struct event_trigger_ops hist_enable_trigger_ops = {
6288         .func                   = hist_enable_trigger,
6289         .print                  = event_enable_trigger_print,
6290         .init                   = event_trigger_init,
6291         .free                   = event_enable_trigger_free,
6292 };
6293
6294 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6295         .func                   = hist_enable_count_trigger,
6296         .print                  = event_enable_trigger_print,
6297         .init                   = event_trigger_init,
6298         .free                   = event_enable_trigger_free,
6299 };
6300
6301 static struct event_trigger_ops hist_disable_trigger_ops = {
6302         .func                   = hist_enable_trigger,
6303         .print                  = event_enable_trigger_print,
6304         .init                   = event_trigger_init,
6305         .free                   = event_enable_trigger_free,
6306 };
6307
6308 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6309         .func                   = hist_enable_count_trigger,
6310         .print                  = event_enable_trigger_print,
6311         .init                   = event_trigger_init,
6312         .free                   = event_enable_trigger_free,
6313 };
6314
6315 static struct event_trigger_ops *
6316 hist_enable_get_trigger_ops(char *cmd, char *param)
6317 {
6318         struct event_trigger_ops *ops;
6319         bool enable;
6320
6321         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6322
6323         if (enable)
6324                 ops = param ? &hist_enable_count_trigger_ops :
6325                         &hist_enable_trigger_ops;
6326         else
6327                 ops = param ? &hist_disable_count_trigger_ops :
6328                         &hist_disable_trigger_ops;
6329
6330         return ops;
6331 }
6332
6333 static void hist_enable_unreg_all(struct trace_event_file *file)
6334 {
6335         struct event_trigger_data *test, *n;
6336
6337         list_for_each_entry_safe(test, n, &file->triggers, list) {
6338                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6339                         list_del_rcu(&test->list);
6340                         update_cond_flag(file);
6341                         trace_event_trigger_enable_disable(file, 0);
6342                         if (test->ops->free)
6343                                 test->ops->free(test->ops, test);
6344                 }
6345         }
6346 }
6347
6348 static struct event_command trigger_hist_enable_cmd = {
6349         .name                   = ENABLE_HIST_STR,
6350         .trigger_type           = ETT_HIST_ENABLE,
6351         .func                   = event_enable_trigger_func,
6352         .reg                    = event_enable_register_trigger,
6353         .unreg                  = event_enable_unregister_trigger,
6354         .unreg_all              = hist_enable_unreg_all,
6355         .get_trigger_ops        = hist_enable_get_trigger_ops,
6356         .set_filter             = set_trigger_filter,
6357 };
6358
6359 static struct event_command trigger_hist_disable_cmd = {
6360         .name                   = DISABLE_HIST_STR,
6361         .trigger_type           = ETT_HIST_ENABLE,
6362         .func                   = event_enable_trigger_func,
6363         .reg                    = event_enable_register_trigger,
6364         .unreg                  = event_enable_unregister_trigger,
6365         .unreg_all              = hist_enable_unreg_all,
6366         .get_trigger_ops        = hist_enable_get_trigger_ops,
6367         .set_filter             = set_trigger_filter,
6368 };
6369
6370 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6371 {
6372         unregister_event_command(&trigger_hist_enable_cmd);
6373         unregister_event_command(&trigger_hist_disable_cmd);
6374 }
6375
6376 __init int register_trigger_hist_enable_disable_cmds(void)
6377 {
6378         int ret;
6379
6380         ret = register_event_command(&trigger_hist_enable_cmd);
6381         if (WARN_ON(ret < 0))
6382                 return ret;
6383         ret = register_event_command(&trigger_hist_disable_cmd);
6384         if (WARN_ON(ret < 0))
6385                 unregister_trigger_hist_enable_disable_cmds();
6386
6387         return ret;
6388 }
6389
6390 static __init int trace_events_hist_init(void)
6391 {
6392         struct dentry *entry = NULL;
6393         struct dentry *d_tracer;
6394         int err = 0;
6395
6396         err = dyn_event_register(&synth_event_ops);
6397         if (err) {
6398                 pr_warn("Could not register synth_event_ops\n");
6399                 return err;
6400         }
6401
6402         d_tracer = tracing_init_dentry();
6403         if (IS_ERR(d_tracer)) {
6404                 err = PTR_ERR(d_tracer);
6405                 goto err;
6406         }
6407
6408         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
6409                                     NULL, &synth_events_fops);
6410         if (!entry) {
6411                 err = -ENODEV;
6412                 goto err;
6413         }
6414
6415         return err;
6416  err:
6417         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
6418
6419         return err;
6420 }
6421
6422 fs_initcall(trace_events_hist_init);