kernel/trace/trace_events_hist.c

   1 /*
   2  * trace_events_hist - trace event hist triggers
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
  15  */
  16
  17 #include <linux/module.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/mutex.h>
  20 #include <linux/slab.h>
  21 #include <linux/stacktrace.h>
  22
  23 #include "tracing_map.h"
  24 #include "trace.h"
  25
  26 struct hist_field;
  27
  28 typedef u64 (*hist_field_fn_t) (struct hist_field *field, void *event);
  29
  30 struct hist_field {
  31         struct ftrace_event_field       *field;
  32         unsigned long                   flags;
  33         hist_field_fn_t                 fn;
  34         unsigned int                    size;
  35         unsigned int                    offset;
  36 };
  37
  38 static u64 hist_field_counter(struct hist_field *field, void *event)
  39 {
  40         return 1;
  41 }
  42
  43 static u64 hist_field_string(struct hist_field *hist_field, void *event)
  44 {
  45         char *addr = (char *)(event + hist_field->field->offset);
  46
  47         return (u64)(unsigned long)addr;
  48 }
  49
  50 #define DEFINE_HIST_FIELD_FN(type)                                      \
  51 static u64 hist_field_##type(struct hist_field *hist_field, void *event)\
  52 {                                                                       \
  53         type *addr = (type *)(event + hist_field->field->offset);       \
  54                                                                         \
  55         return (u64)*addr;                                              \
  56 }
  57
  58 DEFINE_HIST_FIELD_FN(s64);
  59 DEFINE_HIST_FIELD_FN(u64);
  60 DEFINE_HIST_FIELD_FN(s32);
  61 DEFINE_HIST_FIELD_FN(u32);
  62 DEFINE_HIST_FIELD_FN(s16);
  63 DEFINE_HIST_FIELD_FN(u16);
  64 DEFINE_HIST_FIELD_FN(s8);
  65 DEFINE_HIST_FIELD_FN(u8);
  66
  67 #define for_each_hist_field(i, hist_data)       \
  68         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
  69
  70 #define for_each_hist_val_field(i, hist_data)   \
  71         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
  72
  73 #define for_each_hist_key_field(i, hist_data)   \
  74         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
  75
  76 #define HITCOUNT_IDX            0
  77 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + sizeof(u64))
  78
  79 enum hist_field_flags {
  80         HIST_FIELD_FL_HITCOUNT          = 1,
  81         HIST_FIELD_FL_KEY               = 2,
  82         HIST_FIELD_FL_STRING            = 4,
  83         HIST_FIELD_FL_HEX               = 8,
  84         HIST_FIELD_FL_SYM               = 16,
  85         HIST_FIELD_FL_SYM_OFFSET        = 32,
  86         HIST_FIELD_FL_EXECNAME          = 64,
  87 };
  88
  89 struct hist_trigger_attrs {
  90         char            *keys_str;
  91         char            *vals_str;
  92         char            *sort_key_str;
  93         bool            pause;
  94         bool            cont;
  95         bool            clear;
  96         unsigned int    map_bits;
  97 };
  98
  99 struct hist_trigger_data {
 100         struct hist_field               *fields[TRACING_MAP_FIELDS_MAX];
 101         unsigned int                    n_vals;
 102         unsigned int                    n_keys;
 103         unsigned int                    n_fields;
 104         unsigned int                    key_size;
 105         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 106         unsigned int                    n_sort_keys;
 107         struct trace_event_file         *event_file;
 108         struct hist_trigger_attrs       *attrs;
 109         struct tracing_map              *map;
 110 };
 111
 112 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
 113 {
 114         hist_field_fn_t fn = NULL;
 115
 116         switch (field_size) {
 117         case 8:
 118                 if (field_is_signed)
 119                         fn = hist_field_s64;
 120                 else
 121                         fn = hist_field_u64;
 122                 break;
 123         case 4:
 124                 if (field_is_signed)
 125                         fn = hist_field_s32;
 126                 else
 127                         fn = hist_field_u32;
 128                 break;
 129         case 2:
 130                 if (field_is_signed)
 131                         fn = hist_field_s16;
 132                 else
 133                         fn = hist_field_u16;
 134                 break;
 135         case 1:
 136                 if (field_is_signed)
 137                         fn = hist_field_s8;
 138                 else
 139                         fn = hist_field_u8;
 140                 break;
 141         }
 142
 143         return fn;
 144 }
 145
 146 static int parse_map_size(char *str)
 147 {
 148         unsigned long size, map_bits;
 149         int ret;
 150
 151         strsep(&str, "=");
 152         if (!str) {
 153                 ret = -EINVAL;
 154                 goto out;
 155         }
 156
 157         ret = kstrtoul(str, 0, &size);
 158         if (ret)
 159                 goto out;
 160
 161         map_bits = ilog2(roundup_pow_of_two(size));
 162         if (map_bits < TRACING_MAP_BITS_MIN ||
 163             map_bits > TRACING_MAP_BITS_MAX)
 164                 ret = -EINVAL;
 165         else
 166                 ret = map_bits;
 167  out:
 168         return ret;
 169 }
 170
 171 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
 172 {
 173         if (!attrs)
 174                 return;
 175
 176         kfree(attrs->sort_key_str);
 177         kfree(attrs->keys_str);
 178         kfree(attrs->vals_str);
 179         kfree(attrs);
 180 }
 181
 182 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
 183 {
 184         struct hist_trigger_attrs *attrs;
 185         int ret = 0;
 186
 187         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
 188         if (!attrs)
 189                 return ERR_PTR(-ENOMEM);
 190
 191         while (trigger_str) {
 192                 char *str = strsep(&trigger_str, ":");
 193
 194                 if ((strncmp(str, "key=", strlen("key=")) == 0) ||
 195                     (strncmp(str, "keys=", strlen("keys=")) == 0))
 196                         attrs->keys_str = kstrdup(str, GFP_KERNEL);
 197                 else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
 198                          (strncmp(str, "vals=", strlen("vals=")) == 0) ||
 199                          (strncmp(str, "values=", strlen("values=")) == 0))
 200                         attrs->vals_str = kstrdup(str, GFP_KERNEL);
 201                 else if (strncmp(str, "sort=", strlen("sort=")) == 0)
 202                         attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
 203                 else if (strcmp(str, "pause") == 0)
 204                         attrs->pause = true;
 205                 else if ((strcmp(str, "cont") == 0) ||
 206                          (strcmp(str, "continue") == 0))
 207                         attrs->cont = true;
 208                 else if (strcmp(str, "clear") == 0)
 209                         attrs->clear = true;
 210                 else if (strncmp(str, "size=", strlen("size=")) == 0) {
 211                         int map_bits = parse_map_size(str);
 212
 213                         if (map_bits < 0) {
 214                                 ret = map_bits;
 215                                 goto free;
 216                         }
 217                         attrs->map_bits = map_bits;
 218                 } else {
 219                         ret = -EINVAL;
 220                         goto free;
 221                 }
 222         }
 223
 224         if (!attrs->keys_str) {
 225                 ret = -EINVAL;
 226                 goto free;
 227         }
 228
 229         return attrs;
 230  free:
 231         destroy_hist_trigger_attrs(attrs);
 232
 233         return ERR_PTR(ret);
 234 }
 235
 236 static inline void save_comm(char *comm, struct task_struct *task)
 237 {
 238         if (!task->pid) {
 239                 strcpy(comm, "<idle>");
 240                 return;
 241         }
 242
 243         if (WARN_ON_ONCE(task->pid < 0)) {
 244                 strcpy(comm, "<XXX>");
 245                 return;
 246         }
 247
 248         memcpy(comm, task->comm, TASK_COMM_LEN);
 249 }
 250
 251 static void hist_trigger_elt_comm_free(struct tracing_map_elt *elt)
 252 {
 253         kfree((char *)elt->private_data);
 254 }
 255
 256 static int hist_trigger_elt_comm_alloc(struct tracing_map_elt *elt)
 257 {
 258         struct hist_trigger_data *hist_data = elt->map->private_data;
 259         struct hist_field *key_field;
 260         unsigned int i;
 261
 262         for_each_hist_key_field(i, hist_data) {
 263                 key_field = hist_data->fields[i];
 264
 265                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
 266                         unsigned int size = TASK_COMM_LEN + 1;
 267
 268                         elt->private_data = kzalloc(size, GFP_KERNEL);
 269                         if (!elt->private_data)
 270                                 return -ENOMEM;
 271                         break;
 272                 }
 273         }
 274
 275         return 0;
 276 }
 277
 278 static void hist_trigger_elt_comm_copy(struct tracing_map_elt *to,
 279                                        struct tracing_map_elt *from)
 280 {
 281         char *comm_from = from->private_data;
 282         char *comm_to = to->private_data;
 283
 284         if (comm_from)
 285                 memcpy(comm_to, comm_from, TASK_COMM_LEN + 1);
 286 }
 287
 288 static void hist_trigger_elt_comm_init(struct tracing_map_elt *elt)
 289 {
 290         char *comm = elt->private_data;
 291
 292         if (comm)
 293                 save_comm(comm, current);
 294 }
 295
 296 static const struct tracing_map_ops hist_trigger_elt_comm_ops = {
 297         .elt_alloc      = hist_trigger_elt_comm_alloc,
 298         .elt_copy       = hist_trigger_elt_comm_copy,
 299         .elt_free       = hist_trigger_elt_comm_free,
 300         .elt_init       = hist_trigger_elt_comm_init,
 301 };
 302
 303 static void destroy_hist_field(struct hist_field *hist_field)
 304 {
 305         kfree(hist_field);
 306 }
 307
 308 static struct hist_field *create_hist_field(struct ftrace_event_field *field,
 309                                             unsigned long flags)
 310 {
 311         struct hist_field *hist_field;
 312
 313         if (field && is_function_field(field))
 314                 return NULL;
 315
 316         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
 317         if (!hist_field)
 318                 return NULL;
 319
 320         if (flags & HIST_FIELD_FL_HITCOUNT) {
 321                 hist_field->fn = hist_field_counter;
 322                 goto out;
 323         }
 324
 325         if (is_string_field(field)) {
 326                 flags |= HIST_FIELD_FL_STRING;
 327                 hist_field->fn = hist_field_string;
 328         } else {
 329                 hist_field->fn = select_value_fn(field->size,
 330                                                  field->is_signed);
 331                 if (!hist_field->fn) {
 332                         destroy_hist_field(hist_field);
 333                         return NULL;
 334                 }
 335         }
 336  out:
 337         hist_field->field = field;
 338         hist_field->flags = flags;
 339
 340         return hist_field;
 341 }
 342
 343 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
 344 {
 345         unsigned int i;
 346
 347         for (i = 0; i < TRACING_MAP_FIELDS_MAX; i++) {
 348                 if (hist_data->fields[i]) {
 349                         destroy_hist_field(hist_data->fields[i]);
 350                         hist_data->fields[i] = NULL;
 351                 }
 352         }
 353 }
 354
 355 static int create_hitcount_val(struct hist_trigger_data *hist_data)
 356 {
 357         hist_data->fields[HITCOUNT_IDX] =
 358                 create_hist_field(NULL, HIST_FIELD_FL_HITCOUNT);
 359         if (!hist_data->fields[HITCOUNT_IDX])
 360                 return -ENOMEM;
 361
 362         hist_data->n_vals++;
 363
 364         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
 365                 return -EINVAL;
 366
 367         return 0;
 368 }
 369
 370 static int create_val_field(struct hist_trigger_data *hist_data,
 371                             unsigned int val_idx,
 372                             struct trace_event_file *file,
 373                             char *field_str)
 374 {
 375         struct ftrace_event_field *field = NULL;
 376         unsigned long flags = 0;
 377         char *field_name;
 378         int ret = 0;
 379
 380         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
 381                 return -EINVAL;
 382
 383         field_name = strsep(&field_str, ".");
 384         if (field_str) {
 385                 if (strcmp(field_str, "hex") == 0)
 386                         flags |= HIST_FIELD_FL_HEX;
 387                 else {
 388                         ret = -EINVAL;
 389                         goto out;
 390                 }
 391         }
 392
 393         field = trace_find_event_field(file->event_call, field_name);
 394         if (!field) {
 395                 ret = -EINVAL;
 396                 goto out;
 397         }
 398
 399         hist_data->fields[val_idx] = create_hist_field(field, flags);
 400         if (!hist_data->fields[val_idx]) {
 401                 ret = -ENOMEM;
 402                 goto out;
 403         }
 404
 405         ++hist_data->n_vals;
 406
 407         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
 408                 ret = -EINVAL;
 409  out:
 410         return ret;
 411 }
 412
 413 static int create_val_fields(struct hist_trigger_data *hist_data,
 414                              struct trace_event_file *file)
 415 {
 416         char *fields_str, *field_str;
 417         unsigned int i, j;
 418         int ret;
 419
 420         ret = create_hitcount_val(hist_data);
 421         if (ret)
 422                 goto out;
 423
 424         fields_str = hist_data->attrs->vals_str;
 425         if (!fields_str)
 426                 goto out;
 427
 428         strsep(&fields_str, "=");
 429         if (!fields_str)
 430                 goto out;
 431
 432         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
 433                      j < TRACING_MAP_VALS_MAX; i++) {
 434                 field_str = strsep(&fields_str, ",");
 435                 if (!field_str)
 436                         break;
 437                 if (strcmp(field_str, "hitcount") == 0)
 438                         continue;
 439                 ret = create_val_field(hist_data, j++, file, field_str);
 440                 if (ret)
 441                         goto out;
 442         }
 443         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
 444                 ret = -EINVAL;
 445  out:
 446         return ret;
 447 }
 448
 449 static int create_key_field(struct hist_trigger_data *hist_data,
 450                             unsigned int key_idx,
 451                             unsigned int key_offset,
 452                             struct trace_event_file *file,
 453                             char *field_str)
 454 {
 455         struct ftrace_event_field *field = NULL;
 456         unsigned long flags = 0;
 457         unsigned int key_size;
 458         char *field_name;
 459         int ret = 0;
 460
 461         if (WARN_ON(key_idx >= TRACING_MAP_FIELDS_MAX))
 462                 return -EINVAL;
 463
 464         flags |= HIST_FIELD_FL_KEY;
 465
 466         field_name = strsep(&field_str, ".");
 467         if (field_str) {
 468                 if (strcmp(field_str, "hex") == 0)
 469                         flags |= HIST_FIELD_FL_HEX;
 470                 else if (strcmp(field_str, "sym") == 0)
 471                         flags |= HIST_FIELD_FL_SYM;
 472                 else if (strcmp(field_str, "sym-offset") == 0)
 473                         flags |= HIST_FIELD_FL_SYM_OFFSET;
 474                 else if ((strcmp(field_str, "execname") == 0) &&
 475                          (strcmp(field_name, "common_pid") == 0))
 476                         flags |= HIST_FIELD_FL_EXECNAME;
 477                 else {
 478                         ret = -EINVAL;
 479                         goto out;
 480                 }
 481         }
 482
 483         field = trace_find_event_field(file->event_call, field_name);
 484         if (!field) {
 485                 ret = -EINVAL;
 486                 goto out;
 487         }
 488
 489         key_size = field->size;
 490
 491         hist_data->fields[key_idx] = create_hist_field(field, flags);
 492         if (!hist_data->fields[key_idx]) {
 493                 ret = -ENOMEM;
 494                 goto out;
 495         }
 496
 497         key_size = ALIGN(key_size, sizeof(u64));
 498         hist_data->fields[key_idx]->size = key_size;
 499         hist_data->fields[key_idx]->offset = key_offset;
 500         hist_data->key_size += key_size;
 501         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
 502                 ret = -EINVAL;
 503                 goto out;
 504         }
 505
 506         hist_data->n_keys++;
 507
 508         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
 509                 return -EINVAL;
 510
 511         ret = key_size;
 512  out:
 513         return ret;
 514 }
 515
 516 static int create_key_fields(struct hist_trigger_data *hist_data,
 517                              struct trace_event_file *file)
 518 {
 519         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
 520         char *fields_str, *field_str;
 521         int ret = -EINVAL;
 522
 523         fields_str = hist_data->attrs->keys_str;
 524         if (!fields_str)
 525                 goto out;
 526
 527         strsep(&fields_str, "=");
 528         if (!fields_str)
 529                 goto out;
 530
 531         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
 532                 field_str = strsep(&fields_str, ",");
 533                 if (!field_str)
 534                         break;
 535                 ret = create_key_field(hist_data, i, key_offset,
 536                                        file, field_str);
 537                 if (ret < 0)
 538                         goto out;
 539                 key_offset += ret;
 540         }
 541         if (fields_str) {
 542                 ret = -EINVAL;
 543                 goto out;
 544         }
 545         ret = 0;
 546  out:
 547         return ret;
 548 }
 549
 550 static int create_hist_fields(struct hist_trigger_data *hist_data,
 551                               struct trace_event_file *file)
 552 {
 553         int ret;
 554
 555         ret = create_val_fields(hist_data, file);
 556         if (ret)
 557                 goto out;
 558
 559         ret = create_key_fields(hist_data, file);
 560         if (ret)
 561                 goto out;
 562
 563         hist_data->n_fields = hist_data->n_vals + hist_data->n_keys;
 564  out:
 565         return ret;
 566 }
 567
 568 static int is_descending(const char *str)
 569 {
 570         if (!str)
 571                 return 0;
 572
 573         if (strcmp(str, "descending") == 0)
 574                 return 1;
 575
 576         if (strcmp(str, "ascending") == 0)
 577                 return 0;
 578
 579         return -EINVAL;
 580 }
 581
 582 static int create_sort_keys(struct hist_trigger_data *hist_data)
 583 {
 584         char *fields_str = hist_data->attrs->sort_key_str;
 585         struct ftrace_event_field *field = NULL;
 586         struct tracing_map_sort_key *sort_key;
 587         int descending, ret = 0;
 588         unsigned int i, j;
 589
 590         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
 591
 592         if (!fields_str)
 593                 goto out;
 594
 595         strsep(&fields_str, "=");
 596         if (!fields_str) {
 597                 ret = -EINVAL;
 598                 goto out;
 599         }
 600
 601         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
 602                 char *field_str, *field_name;
 603
 604                 sort_key = &hist_data->sort_keys[i];
 605
 606                 field_str = strsep(&fields_str, ",");
 607                 if (!field_str) {
 608                         if (i == 0)
 609                                 ret = -EINVAL;
 610                         break;
 611                 }
 612
 613                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
 614                         ret = -EINVAL;
 615                         break;
 616                 }
 617
 618                 field_name = strsep(&field_str, ".");
 619                 if (!field_name) {
 620                         ret = -EINVAL;
 621                         break;
 622                 }
 623
 624                 if (strcmp(field_name, "hitcount") == 0) {
 625                         descending = is_descending(field_str);
 626                         if (descending < 0) {
 627                                 ret = descending;
 628                                 break;
 629                         }
 630                         sort_key->descending = descending;
 631                         continue;
 632                 }
 633
 634                 for (j = 1; j < hist_data->n_fields; j++) {
 635                         field = hist_data->fields[j]->field;
 636                         if (field && (strcmp(field_name, field->name) == 0)) {
 637                                 sort_key->field_idx = j;
 638                                 descending = is_descending(field_str);
 639                                 if (descending < 0) {
 640                                         ret = descending;
 641                                         goto out;
 642                                 }
 643                                 sort_key->descending = descending;
 644                                 break;
 645                         }
 646                 }
 647                 if (j == hist_data->n_fields) {
 648                         ret = -EINVAL;
 649                         break;
 650                 }
 651         }
 652         hist_data->n_sort_keys = i;
 653  out:
 654         return ret;
 655 }
 656
 657 static void destroy_hist_data(struct hist_trigger_data *hist_data)
 658 {
 659         destroy_hist_trigger_attrs(hist_data->attrs);
 660         destroy_hist_fields(hist_data);
 661         tracing_map_destroy(hist_data->map);
 662         kfree(hist_data);
 663 }
 664
 665 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
 666 {
 667         struct tracing_map *map = hist_data->map;
 668         struct ftrace_event_field *field;
 669         struct hist_field *hist_field;
 670         unsigned int i, idx;
 671
 672         for_each_hist_field(i, hist_data) {
 673                 hist_field = hist_data->fields[i];
 674                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
 675                         tracing_map_cmp_fn_t cmp_fn;
 676
 677                         field = hist_field->field;
 678
 679                         if (is_string_field(field))
 680                                 cmp_fn = tracing_map_cmp_string;
 681                         else
 682                                 cmp_fn = tracing_map_cmp_num(field->size,
 683                                                              field->is_signed);
 684                         idx = tracing_map_add_key_field(map,
 685                                                         hist_field->offset,
 686                                                         cmp_fn);
 687
 688                 } else
 689                         idx = tracing_map_add_sum_field(map);
 690
 691                 if (idx < 0)
 692                         return idx;
 693         }
 694
 695         return 0;
 696 }
 697
 698 static bool need_tracing_map_ops(struct hist_trigger_data *hist_data)
 699 {
 700         struct hist_field *key_field;
 701         unsigned int i;
 702
 703         for_each_hist_key_field(i, hist_data) {
 704                 key_field = hist_data->fields[i];
 705
 706                 if (key_field->flags & HIST_FIELD_FL_EXECNAME)
 707                         return true;
 708         }
 709
 710         return false;
 711 }
 712
 713 static struct hist_trigger_data *
 714 create_hist_data(unsigned int map_bits,
 715                  struct hist_trigger_attrs *attrs,
 716                  struct trace_event_file *file)
 717 {
 718         const struct tracing_map_ops *map_ops = NULL;
 719         struct hist_trigger_data *hist_data;
 720         int ret = 0;
 721
 722         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
 723         if (!hist_data)
 724                 return ERR_PTR(-ENOMEM);
 725
 726         hist_data->attrs = attrs;
 727
 728         ret = create_hist_fields(hist_data, file);
 729         if (ret)
 730                 goto free;
 731
 732         ret = create_sort_keys(hist_data);
 733         if (ret)
 734                 goto free;
 735
 736         if (need_tracing_map_ops(hist_data))
 737                 map_ops = &hist_trigger_elt_comm_ops;
 738
 739         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
 740                                             map_ops, hist_data);
 741         if (IS_ERR(hist_data->map)) {
 742                 ret = PTR_ERR(hist_data->map);
 743                 hist_data->map = NULL;
 744                 goto free;
 745         }
 746
 747         ret = create_tracing_map_fields(hist_data);
 748         if (ret)
 749                 goto free;
 750
 751         ret = tracing_map_init(hist_data->map);
 752         if (ret)
 753                 goto free;
 754
 755         hist_data->event_file = file;
 756  out:
 757         return hist_data;
 758  free:
 759         hist_data->attrs = NULL;
 760
 761         destroy_hist_data(hist_data);
 762
 763         hist_data = ERR_PTR(ret);
 764
 765         goto out;
 766 }
 767
 768 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
 769                                     struct tracing_map_elt *elt,
 770                                     void *rec)
 771 {
 772         struct hist_field *hist_field;
 773         unsigned int i;
 774         u64 hist_val;
 775
 776         for_each_hist_val_field(i, hist_data) {
 777                 hist_field = hist_data->fields[i];
 778                 hist_val = hist_field->fn(hist_field, rec);
 779                 tracing_map_update_sum(elt, i, hist_val);
 780         }
 781 }
 782
 783 static void event_hist_trigger(struct event_trigger_data *data, void *rec)
 784 {
 785         struct hist_trigger_data *hist_data = data->private_data;
 786         char compound_key[HIST_KEY_SIZE_MAX];
 787         struct hist_field *key_field;
 788         struct tracing_map_elt *elt;
 789         u64 field_contents;
 790         void *key = NULL;
 791         unsigned int i;
 792
 793         if (hist_data->n_keys > 1)
 794                 memset(compound_key, 0, hist_data->key_size);
 795
 796         for_each_hist_key_field(i, hist_data) {
 797                 key_field = hist_data->fields[i];
 798
 799                 field_contents = key_field->fn(key_field, rec);
 800                 if (key_field->flags & HIST_FIELD_FL_STRING)
 801                         key = (void *)(unsigned long)field_contents;
 802                 else
 803                         key = (void *)&field_contents;
 804
 805                 if (hist_data->n_keys > 1) {
 806                         memcpy(compound_key + key_field->offset, key,
 807                                key_field->size);
 808                 }
 809         }
 810
 811         if (hist_data->n_keys > 1)
 812                 key = compound_key;
 813
 814         elt = tracing_map_insert(hist_data->map, key);
 815         if (elt)
 816                 hist_trigger_elt_update(hist_data, elt, rec);
 817 }
 818
 819 static void
 820 hist_trigger_entry_print(struct seq_file *m,
 821                          struct hist_trigger_data *hist_data, void *key,
 822                          struct tracing_map_elt *elt)
 823 {
 824         struct hist_field *key_field;
 825         char str[KSYM_SYMBOL_LEN];
 826         unsigned int i;
 827         u64 uval;
 828
 829         seq_puts(m, "{ ");
 830
 831         for_each_hist_key_field(i, hist_data) {
 832                 key_field = hist_data->fields[i];
 833
 834                 if (i > hist_data->n_vals)
 835                         seq_puts(m, ", ");
 836
 837                 if (key_field->flags & HIST_FIELD_FL_HEX) {
 838                         uval = *(u64 *)(key + key_field->offset);
 839                         seq_printf(m, "%s: %llx",
 840                                    key_field->field->name, uval);
 841                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
 842                         uval = *(u64 *)(key + key_field->offset);
 843                         sprint_symbol_no_offset(str, uval);
 844                         seq_printf(m, "%s: [%llx] %-45s",
 845                                    key_field->field->name, uval, str);
 846                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
 847                         uval = *(u64 *)(key + key_field->offset);
 848                         sprint_symbol(str, uval);
 849                         seq_printf(m, "%s: [%llx] %-55s",
 850                                    key_field->field->name, uval, str);
 851                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
 852                         char *comm = elt->private_data;
 853
 854                         uval = *(u64 *)(key + key_field->offset);
 855                         seq_printf(m, "%s: %-16s[%10llu]",
 856                                    key_field->field->name, comm, uval);
 857                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
 858                         seq_printf(m, "%s: %-50s", key_field->field->name,
 859                                    (char *)(key + key_field->offset));
 860                 } else {
 861                         uval = *(u64 *)(key + key_field->offset);
 862                         seq_printf(m, "%s: %10llu", key_field->field->name,
 863                                    uval);
 864                 }
 865         }
 866
 867         seq_puts(m, " }");
 868
 869         seq_printf(m, " hitcount: %10llu",
 870                    tracing_map_read_sum(elt, HITCOUNT_IDX));
 871
 872         for (i = 1; i < hist_data->n_vals; i++) {
 873                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
 874                         seq_printf(m, "  %s: %10llx",
 875                                    hist_data->fields[i]->field->name,
 876                                    tracing_map_read_sum(elt, i));
 877                 } else {
 878                         seq_printf(m, "  %s: %10llu",
 879                                    hist_data->fields[i]->field->name,
 880                                    tracing_map_read_sum(elt, i));
 881                 }
 882         }
 883
 884         seq_puts(m, "\n");
 885 }
 886
 887 static int print_entries(struct seq_file *m,
 888                          struct hist_trigger_data *hist_data)
 889 {
 890         struct tracing_map_sort_entry **sort_entries = NULL;
 891         struct tracing_map *map = hist_data->map;
 892         unsigned int i, n_entries;
 893
 894         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
 895                                              hist_data->n_sort_keys,
 896                                              &sort_entries);
 897         if (n_entries < 0)
 898                 return n_entries;
 899
 900         for (i = 0; i < n_entries; i++)
 901                 hist_trigger_entry_print(m, hist_data,
 902                                          sort_entries[i]->key,
 903                                          sort_entries[i]->elt);
 904
 905         tracing_map_destroy_sort_entries(sort_entries, n_entries);
 906
 907         return n_entries;
 908 }
 909
 910 static int hist_show(struct seq_file *m, void *v)
 911 {
 912         struct event_trigger_data *test, *data = NULL;
 913         struct trace_event_file *event_file;
 914         struct hist_trigger_data *hist_data;
 915         int n_entries, ret = 0;
 916
 917         mutex_lock(&event_mutex);
 918
 919         event_file = event_file_data(m->private);
 920         if (unlikely(!event_file)) {
 921                 ret = -ENODEV;
 922                 goto out_unlock;
 923         }
 924
 925         list_for_each_entry_rcu(test, &event_file->triggers, list) {
 926                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 927                         data = test;
 928                         break;
 929                 }
 930         }
 931         if (!data)
 932                 goto out_unlock;
 933
 934         seq_puts(m, "# event histogram\n#\n# trigger info: ");
 935         data->ops->print(m, data->ops, data);
 936         seq_puts(m, "\n");
 937
 938         hist_data = data->private_data;
 939         n_entries = print_entries(m, hist_data);
 940         if (n_entries < 0) {
 941                 ret = n_entries;
 942                 n_entries = 0;
 943         }
 944
 945         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
 946                    (u64)atomic64_read(&hist_data->map->hits),
 947                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
 948  out_unlock:
 949         mutex_unlock(&event_mutex);
 950
 951         return ret;
 952 }
 953
 954 static int event_hist_open(struct inode *inode, struct file *file)
 955 {
 956         return single_open(file, hist_show, file);
 957 }
 958
 959 const struct file_operations event_hist_fops = {
 960         .open = event_hist_open,
 961         .read = seq_read,
 962         .llseek = seq_lseek,
 963         .release = single_release,
 964 };
 965
 966 static const char *get_hist_field_flags(struct hist_field *hist_field)
 967 {
 968         const char *flags_str = NULL;
 969
 970         if (hist_field->flags & HIST_FIELD_FL_HEX)
 971                 flags_str = "hex";
 972         else if (hist_field->flags & HIST_FIELD_FL_SYM)
 973                 flags_str = "sym";
 974         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
 975                 flags_str = "sym-offset";
 976         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
 977                 flags_str = "execname";
 978
 979         return flags_str;
 980 }
 981
 982 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
 983 {
 984         seq_printf(m, "%s", hist_field->field->name);
 985         if (hist_field->flags) {
 986                 const char *flags_str = get_hist_field_flags(hist_field);
 987
 988                 if (flags_str)
 989                         seq_printf(m, ".%s", flags_str);
 990         }
 991 }
 992
 993 static int event_hist_trigger_print(struct seq_file *m,
 994                                     struct event_trigger_ops *ops,
 995                                     struct event_trigger_data *data)
 996 {
 997         struct hist_trigger_data *hist_data = data->private_data;
 998         struct hist_field *key_field;
 999         unsigned int i;
1000
1001         seq_puts(m, "hist:keys=");
1002
1003         for_each_hist_key_field(i, hist_data) {
1004                 key_field = hist_data->fields[i];
1005
1006                 if (i > hist_data->n_vals)
1007                         seq_puts(m, ",");
1008
1009                 hist_field_print(m, key_field);
1010         }
1011
1012         seq_puts(m, ":vals=");
1013
1014         for_each_hist_val_field(i, hist_data) {
1015                 if (i == HITCOUNT_IDX)
1016                         seq_puts(m, "hitcount");
1017                 else {
1018                         seq_puts(m, ",");
1019                         hist_field_print(m, hist_data->fields[i]);
1020                 }
1021         }
1022
1023         seq_puts(m, ":sort=");
1024
1025         for (i = 0; i < hist_data->n_sort_keys; i++) {
1026                 struct tracing_map_sort_key *sort_key;
1027
1028                 sort_key = &hist_data->sort_keys[i];
1029
1030                 if (i > 0)
1031                         seq_puts(m, ",");
1032
1033                 if (sort_key->field_idx == HITCOUNT_IDX)
1034                         seq_puts(m, "hitcount");
1035                 else {
1036                         unsigned int idx = sort_key->field_idx;
1037
1038                         if (WARN_ON(idx >= TRACING_MAP_FIELDS_MAX))
1039                                 return -EINVAL;
1040
1041                         hist_field_print(m, hist_data->fields[idx]);
1042                 }
1043
1044                 if (sort_key->descending)
1045                         seq_puts(m, ".descending");
1046         }
1047
1048         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
1049
1050         if (data->filter_str)
1051                 seq_printf(m, " if %s", data->filter_str);
1052
1053         if (data->paused)
1054                 seq_puts(m, " [paused]");
1055         else
1056                 seq_puts(m, " [active]");
1057
1058         seq_putc(m, '\n');
1059
1060         return 0;
1061 }
1062
1063 static void event_hist_trigger_free(struct event_trigger_ops *ops,
1064                                     struct event_trigger_data *data)
1065 {
1066         struct hist_trigger_data *hist_data = data->private_data;
1067
1068         if (WARN_ON_ONCE(data->ref <= 0))
1069                 return;
1070
1071         data->ref--;
1072         if (!data->ref) {
1073                 trigger_data_free(data);
1074                 destroy_hist_data(hist_data);
1075         }
1076 }
1077
1078 static struct event_trigger_ops event_hist_trigger_ops = {
1079         .func                   = event_hist_trigger,
1080         .print                  = event_hist_trigger_print,
1081         .init                   = event_trigger_init,
1082         .free                   = event_hist_trigger_free,
1083 };
1084
1085 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
1086                                                             char *param)
1087 {
1088         return &event_hist_trigger_ops;
1089 }
1090
1091 static void hist_clear(struct event_trigger_data *data)
1092 {
1093         struct hist_trigger_data *hist_data = data->private_data;
1094         bool paused;
1095
1096         paused = data->paused;
1097         data->paused = true;
1098
1099         synchronize_sched();
1100
1101         tracing_map_clear(hist_data->map);
1102
1103         data->paused = paused;
1104 }
1105
1106 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
1107                                  struct event_trigger_data *data,
1108                                  struct trace_event_file *file)
1109 {
1110         struct hist_trigger_data *hist_data = data->private_data;
1111         struct event_trigger_data *test;
1112         int ret = 0;
1113
1114         list_for_each_entry_rcu(test, &file->triggers, list) {
1115                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1116                         if (hist_data->attrs->pause)
1117                                 test->paused = true;
1118                         else if (hist_data->attrs->cont)
1119                                 test->paused = false;
1120                         else if (hist_data->attrs->clear)
1121                                 hist_clear(test);
1122                         else
1123                                 ret = -EEXIST;
1124                         goto out;
1125                 }
1126         }
1127
1128         if (hist_data->attrs->cont || hist_data->attrs->clear) {
1129                 ret = -ENOENT;
1130                 goto out;
1131         }
1132
1133         if (hist_data->attrs->pause)
1134                 data->paused = true;
1135
1136         if (data->ops->init) {
1137                 ret = data->ops->init(data->ops, data);
1138                 if (ret < 0)
1139                         goto out;
1140         }
1141
1142         list_add_rcu(&data->list, &file->triggers);
1143         ret++;
1144
1145         update_cond_flag(file);
1146         if (trace_event_trigger_enable_disable(file, 1) < 0) {
1147                 list_del_rcu(&data->list);
1148                 update_cond_flag(file);
1149                 ret--;
1150         }
1151  out:
1152         return ret;
1153 }
1154
1155 static int event_hist_trigger_func(struct event_command *cmd_ops,
1156                                    struct trace_event_file *file,
1157                                    char *glob, char *cmd, char *param)
1158 {
1159         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
1160         struct event_trigger_data *trigger_data;
1161         struct hist_trigger_attrs *attrs;
1162         struct event_trigger_ops *trigger_ops;
1163         struct hist_trigger_data *hist_data;
1164         char *trigger;
1165         int ret = 0;
1166
1167         if (!param)
1168                 return -EINVAL;
1169
1170         /* separate the trigger from the filter (k:v [if filter]) */
1171         trigger = strsep(&param, " \t");
1172         if (!trigger)
1173                 return -EINVAL;
1174
1175         attrs = parse_hist_trigger_attrs(trigger);
1176         if (IS_ERR(attrs))
1177                 return PTR_ERR(attrs);
1178
1179         if (attrs->map_bits)
1180                 hist_trigger_bits = attrs->map_bits;
1181
1182         hist_data = create_hist_data(hist_trigger_bits, attrs, file);
1183         if (IS_ERR(hist_data)) {
1184                 destroy_hist_trigger_attrs(attrs);
1185                 return PTR_ERR(hist_data);
1186         }
1187
1188         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
1189
1190         ret = -ENOMEM;
1191         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
1192         if (!trigger_data)
1193                 goto out_free;
1194
1195         trigger_data->count = -1;
1196         trigger_data->ops = trigger_ops;
1197         trigger_data->cmd_ops = cmd_ops;
1198
1199         INIT_LIST_HEAD(&trigger_data->list);
1200         RCU_INIT_POINTER(trigger_data->filter, NULL);
1201
1202         trigger_data->private_data = hist_data;
1203
1204         if (glob[0] == '!') {
1205                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
1206                 ret = 0;
1207                 goto out_free;
1208         }
1209
1210         if (!param) /* if param is non-empty, it's supposed to be a filter */
1211                 goto out_reg;
1212
1213         if (!cmd_ops->set_filter)
1214                 goto out_reg;
1215
1216         ret = cmd_ops->set_filter(param, trigger_data, file);
1217         if (ret < 0)
1218                 goto out_free;
1219  out_reg:
1220         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
1221         /*
1222          * The above returns on success the # of triggers registered,
1223          * but if it didn't register any it returns zero.  Consider no
1224          * triggers registered a failure too.
1225          */
1226         if (!ret) {
1227                 if (!(attrs->pause || attrs->cont || attrs->clear))
1228                         ret = -ENOENT;
1229                 goto out_free;
1230         } else if (ret < 0)
1231                 goto out_free;
1232         /* Just return zero, not the number of registered triggers */
1233         ret = 0;
1234  out:
1235         return ret;
1236  out_free:
1237         if (cmd_ops->set_filter)
1238                 cmd_ops->set_filter(NULL, trigger_data, NULL);
1239
1240         kfree(trigger_data);
1241
1242         destroy_hist_data(hist_data);
1243         goto out;
1244 }
1245
1246 static struct event_command trigger_hist_cmd = {
1247         .name                   = "hist",
1248         .trigger_type           = ETT_EVENT_HIST,
1249         .flags                  = EVENT_CMD_FL_NEEDS_REC,
1250         .func                   = event_hist_trigger_func,
1251         .reg                    = hist_register_trigger,
1252         .unreg                  = unregister_trigger,
1253         .get_trigger_ops        = event_hist_get_trigger_ops,
1254         .set_filter             = set_trigger_filter,
1255 };
1256
1257 __init int register_trigger_hist_cmd(void)
1258 {
1259         int ret;
1260
1261         ret = register_event_command(&trigger_hist_cmd);
1262         WARN_ON(ret < 0);
1263
1264         return ret;
1265 }