projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tracing: Pass tracing_map_elt to hist_field accessor functions
[linux-2.6-block.git] / kernel / trace / trace_events_hist.c
CommitLineData
7ef224d1
TZ		 1/*
2 * trace_events_hist - trace event hist triggers
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
15 */
16
17#include <linux/module.h>
18#include <linux/kallsyms.h>
19#include <linux/mutex.h>
20#include <linux/slab.h>
21#include <linux/stacktrace.h>
b2d09103 22#include <linux/rculist.h>
7ef224d1
TZ		 23
24#include "tracing_map.h"
25#include "trace.h"
26
27struct hist_field;
28
df35d93b
TZ		 29typedef u64 (*hist_field_fn_t) (struct hist_field *field,
30 struct tracing_map_elt *elt,
31 struct ring_buffer_event *rbe,
32 void *event);
7ef224d1 33
5819eadd 34#define HIST_FIELD_OPERANDS_MAX 2
30350d65
TZ		 35#define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
36
100719dc
TZ		 37enum field_op_id {
38 FIELD_OP_NONE,
39 FIELD_OP_PLUS,
40 FIELD_OP_MINUS,
41 FIELD_OP_UNARY_MINUS,
42};
43
30350d65
TZ		 44struct hist_var {
45 char *name;
46 struct hist_trigger_data *hist_data;
47 unsigned int idx;
48};
5819eadd 49
7ef224d1
TZ		 50struct hist_field {
51 struct ftrace_event_field *field;
52 unsigned long flags;
53 hist_field_fn_t fn;
54 unsigned int size;
76a3b0c8 55 unsigned int offset;
5819eadd
TZ		 56 unsigned int is_signed;
57 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
b559d003 58 struct hist_trigger_data *hist_data;
30350d65 59 struct hist_var var;
100719dc
TZ		 60 enum field_op_id operator;
61 char *name;
7ef224d1
TZ		 62};
63
df35d93b
TZ		 64static u64 hist_field_none(struct hist_field *field,
65 struct tracing_map_elt *elt,
66 struct ring_buffer_event *rbe,
67 void *event)
69a0200c
TZ		 68{
69 return 0;
70}
71
df35d93b
TZ		 72static u64 hist_field_counter(struct hist_field *field,
73 struct tracing_map_elt *elt,
74 struct ring_buffer_event *rbe,
75 void *event)
7ef224d1
TZ		 76{
77 return 1;
78}
79
df35d93b
TZ		 80static u64 hist_field_string(struct hist_field *hist_field,
81 struct tracing_map_elt *elt,
82 struct ring_buffer_event *rbe,
83 void *event)
7ef224d1
TZ		 84{
85 char *addr = (char *)(event + hist_field->field->offset);
86
87 return (u64)(unsigned long)addr;
88}
89
df35d93b
TZ		 90static u64 hist_field_dynstring(struct hist_field *hist_field,
91 struct tracing_map_elt *elt,
92 struct ring_buffer_event *rbe,
93 void *event)
79e577cb
NK		 94{
95 u32 str_item = *(u32 *)(event + hist_field->field->offset);
96 int str_loc = str_item & 0xffff;
97 char *addr = (char *)(event + str_loc);
98
99 return (u64)(unsigned long)addr;
100}
101
df35d93b
TZ		 102static u64 hist_field_pstring(struct hist_field *hist_field,
103 struct tracing_map_elt *elt,
104 struct ring_buffer_event *rbe,
105 void *event)
79e577cb
NK		 106{
107 char **addr = (char **)(event + hist_field->field->offset);
108
109 return (u64)(unsigned long)*addr;
110}
111
df35d93b
TZ		 112static u64 hist_field_log2(struct hist_field *hist_field,
113 struct tracing_map_elt *elt,
114 struct ring_buffer_event *rbe,
115 void *event)
4b94f5b7 116{
5819eadd
TZ		 117 struct hist_field *operand = hist_field->operands[0];
118
df35d93b 119 u64 val = operand->fn(operand, elt, rbe, event);
4b94f5b7
NK		 120
121 return (u64) ilog2(roundup_pow_of_two(val));
122}
123
df35d93b
TZ		 124static u64 hist_field_plus(struct hist_field *hist_field,
125 struct tracing_map_elt *elt,
126 struct ring_buffer_event *rbe,
127 void *event)
100719dc
TZ		 128{
129 struct hist_field *operand1 = hist_field->operands[0];
130 struct hist_field *operand2 = hist_field->operands[1];
131
df35d93b
TZ		 132 u64 val1 = operand1->fn(operand1, elt, rbe, event);
133 u64 val2 = operand2->fn(operand2, elt, rbe, event);
100719dc
TZ		 134
135 return val1 + val2;
136}
137
df35d93b
TZ		 138static u64 hist_field_minus(struct hist_field *hist_field,
139 struct tracing_map_elt *elt,
140 struct ring_buffer_event *rbe,
141 void *event)
100719dc
TZ		 142{
143 struct hist_field *operand1 = hist_field->operands[0];
144 struct hist_field *operand2 = hist_field->operands[1];
145
df35d93b
TZ		 146 u64 val1 = operand1->fn(operand1, elt, rbe, event);
147 u64 val2 = operand2->fn(operand2, elt, rbe, event);
100719dc
TZ		 148
149 return val1 - val2;
150}
151
df35d93b
TZ		 152static u64 hist_field_unary_minus(struct hist_field *hist_field,
153 struct tracing_map_elt *elt,
154 struct ring_buffer_event *rbe,
155 void *event)
100719dc
TZ		 156{
157 struct hist_field *operand = hist_field->operands[0];
158
df35d93b 159 s64 sval = (s64)operand->fn(operand, elt, rbe, event);
100719dc
TZ		 160 u64 val = (u64)-sval;
161
162 return val;
163}
164
7ef224d1 165#define DEFINE_HIST_FIELD_FN(type) \
fbd302cb 166 static u64 hist_field_##type(struct hist_field *hist_field, \
df35d93b
TZ		 167 struct tracing_map_elt *elt, \
168 struct ring_buffer_event *rbe, \
169 void *event) \
7ef224d1
TZ		 170{ \
171 type *addr = (type *)(event + hist_field->field->offset); \
172 \
79e577cb 173 return (u64)(unsigned long)*addr; \
7ef224d1
TZ		 174}
175
176DEFINE_HIST_FIELD_FN(s64);
177DEFINE_HIST_FIELD_FN(u64);
178DEFINE_HIST_FIELD_FN(s32);
179DEFINE_HIST_FIELD_FN(u32);
180DEFINE_HIST_FIELD_FN(s16);
181DEFINE_HIST_FIELD_FN(u16);
182DEFINE_HIST_FIELD_FN(s8);
183DEFINE_HIST_FIELD_FN(u8);
184
185#define for_each_hist_field(i, hist_data) \
186 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
187
188#define for_each_hist_val_field(i, hist_data) \
189 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
190
191#define for_each_hist_key_field(i, hist_data) \
192 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
193
69a0200c
TZ		 194#define HIST_STACKTRACE_DEPTH 16
195#define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
196#define HIST_STACKTRACE_SKIP 5
197
7ef224d1 198#define HITCOUNT_IDX 0
69a0200c 199#define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
7ef224d1
TZ		 200
201enum hist_field_flags {
0d7a8325
TZ		 202 HIST_FIELD_FL_HITCOUNT = 1 << 0,
203 HIST_FIELD_FL_KEY = 1 << 1,
204 HIST_FIELD_FL_STRING = 1 << 2,
205 HIST_FIELD_FL_HEX = 1 << 3,
206 HIST_FIELD_FL_SYM = 1 << 4,
207 HIST_FIELD_FL_SYM_OFFSET = 1 << 5,
208 HIST_FIELD_FL_EXECNAME = 1 << 6,
209 HIST_FIELD_FL_SYSCALL = 1 << 7,
210 HIST_FIELD_FL_STACKTRACE = 1 << 8,
211 HIST_FIELD_FL_LOG2 = 1 << 9,
ad42febe 212 HIST_FIELD_FL_TIMESTAMP = 1 << 10,
860f9f6b 213 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11,
30350d65 214 HIST_FIELD_FL_VAR = 1 << 12,
100719dc 215 HIST_FIELD_FL_EXPR = 1 << 13,
30350d65
TZ		 216};
217
218struct var_defs {
219 unsigned int n_vars;
220 char *name[TRACING_MAP_VARS_MAX];
221 char *expr[TRACING_MAP_VARS_MAX];
7ef224d1
TZ		 222};
223
224struct hist_trigger_attrs {
225 char *keys_str;
f2606835 226 char *vals_str;
e62347d2 227 char *sort_key_str;
5463bfda 228 char *name;
83e99914
TZ		 229 bool pause;
230 bool cont;
e86ae9ba 231 bool clear;
860f9f6b 232 bool ts_in_usecs;
7ef224d1 233 unsigned int map_bits;
30350d65
TZ		 234
235 char *assignment_str[TRACING_MAP_VARS_MAX];
236 unsigned int n_assignments;
237
238 struct var_defs var_defs;
7ef224d1
TZ		 239};
240
241struct hist_trigger_data {
30350d65 242 struct hist_field *fields[HIST_FIELDS_MAX];
7ef224d1
TZ		 243 unsigned int n_vals;
244 unsigned int n_keys;
245 unsigned int n_fields;
30350d65 246 unsigned int n_vars;
7ef224d1
TZ		 247 unsigned int key_size;
248 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
249 unsigned int n_sort_keys;
250 struct trace_event_file *event_file;
251 struct hist_trigger_attrs *attrs;
252 struct tracing_map *map;
ad42febe 253 bool enable_timestamps;
30350d65 254 bool remove;
7ef224d1
TZ		 255};
256
df35d93b
TZ		 257static u64 hist_field_timestamp(struct hist_field *hist_field,
258 struct tracing_map_elt *elt,
259 struct ring_buffer_event *rbe,
260 void *event)
860f9f6b
TZ		 261{
262 struct hist_trigger_data *hist_data = hist_field->hist_data;
263 struct trace_array *tr = hist_data->event_file->tr;
264
265 u64 ts = ring_buffer_event_time_stamp(rbe);
266
267 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
268 ts = ns2usecs(ts);
269
270 return ts;
271}
272
30350d65
TZ		 273static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
274 const char *var_name)
275{
276 struct hist_field *hist_field, *found = NULL;
277 int i;
278
279 for_each_hist_field(i, hist_data) {
280 hist_field = hist_data->fields[i];
281 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
282 strcmp(hist_field->var.name, var_name) == 0) {
283 found = hist_field;
284 break;
285 }
286 }
287
288 return found;
289}
290
291static struct hist_field *find_var(struct hist_trigger_data *hist_data,
292 struct trace_event_file *file,
293 const char *var_name)
294{
295 struct hist_trigger_data *test_data;
296 struct event_trigger_data *test;
297 struct hist_field *hist_field;
298
299 hist_field = find_var_field(hist_data, var_name);
300 if (hist_field)
301 return hist_field;
302
303 list_for_each_entry_rcu(test, &file->triggers, list) {
304 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
305 test_data = test->private_data;
306 hist_field = find_var_field(test_data, var_name);
307 if (hist_field)
308 return hist_field;
309 }
310 }
311
312 return NULL;
313}
314
af6a29bc
TZ		 315struct hist_elt_data {
316 char *comm;
317};
318
85013256
TZ		 319static const char *hist_field_name(struct hist_field *field,
320 unsigned int level)
321{
322 const char *field_name = "";
323
324 if (level > 1)
325 return field_name;
326
327 if (field->field)
328 field_name = field->field->name;
5819eadd
TZ		 329 else if (field->flags & HIST_FIELD_FL_LOG2)
330 field_name = hist_field_name(field->operands[0], ++level);
ad42febe
TZ		 331 else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
332 field_name = "common_timestamp";
100719dc
TZ		 333 else if (field->flags & HIST_FIELD_FL_EXPR)
334 field_name = field->name;
85013256
TZ		 335
336 if (field_name == NULL)
337 field_name = "";
338
339 return field_name;
340}
341
7ef224d1
TZ		 342static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
343{
344 hist_field_fn_t fn = NULL;
345
346 switch (field_size) {
347 case 8:
348 if (field_is_signed)
349 fn = hist_field_s64;
350 else
351 fn = hist_field_u64;
352 break;
353 case 4:
354 if (field_is_signed)
355 fn = hist_field_s32;
356 else
357 fn = hist_field_u32;
358 break;
359 case 2:
360 if (field_is_signed)
361 fn = hist_field_s16;
362 else
363 fn = hist_field_u16;
364 break;
365 case 1:
366 if (field_is_signed)
367 fn = hist_field_s8;
368 else
369 fn = hist_field_u8;
370 break;
371 }
372
373 return fn;
374}
375
376static int parse_map_size(char *str)
377{
378 unsigned long size, map_bits;
379 int ret;
380
381 strsep(&str, "=");
382 if (!str) {
383 ret = -EINVAL;
384 goto out;
385 }
386
387 ret = kstrtoul(str, 0, &size);
388 if (ret)
389 goto out;
390
391 map_bits = ilog2(roundup_pow_of_two(size));
392 if (map_bits < TRACING_MAP_BITS_MIN ||
393 map_bits > TRACING_MAP_BITS_MAX)
394 ret = -EINVAL;
395 else
396 ret = map_bits;
397 out:
398 return ret;
399}
400
401static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
402{
30350d65
TZ		 403 unsigned int i;
404
7ef224d1
TZ		 405 if (!attrs)
406 return;
407
30350d65
TZ		 408 for (i = 0; i < attrs->n_assignments; i++)
409 kfree(attrs->assignment_str[i]);
410
5463bfda 411 kfree(attrs->name);
e62347d2 412 kfree(attrs->sort_key_str);
7ef224d1 413 kfree(attrs->keys_str);
f2606835 414 kfree(attrs->vals_str);
7ef224d1
TZ		 415 kfree(attrs);
416}
417
9b1ae035
TZ		 418static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
419{
420 int ret = 0;
421
422 if ((strncmp(str, "key=", strlen("key=")) == 0) ||
423 (strncmp(str, "keys=", strlen("keys=")) == 0)) {
424 attrs->keys_str = kstrdup(str, GFP_KERNEL);
425 if (!attrs->keys_str) {
426 ret = -ENOMEM;
427 goto out;
428 }
429 } else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
430 (strncmp(str, "vals=", strlen("vals=")) == 0) ||
431 (strncmp(str, "values=", strlen("values=")) == 0)) {
432 attrs->vals_str = kstrdup(str, GFP_KERNEL);
433 if (!attrs->vals_str) {
434 ret = -ENOMEM;
435 goto out;
436 }
437 } else if (strncmp(str, "sort=", strlen("sort=")) == 0) {
438 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
439 if (!attrs->sort_key_str) {
440 ret = -ENOMEM;
441 goto out;
442 }
443 } else if (strncmp(str, "name=", strlen("name=")) == 0) {
444 attrs->name = kstrdup(str, GFP_KERNEL);
445 if (!attrs->name) {
446 ret = -ENOMEM;
447 goto out;
448 }
449 } else if (strncmp(str, "size=", strlen("size=")) == 0) {
450 int map_bits = parse_map_size(str);
451
452 if (map_bits < 0) {
453 ret = map_bits;
454 goto out;
455 }
456 attrs->map_bits = map_bits;
30350d65
TZ		 457 } else {
458 char *assignment;
459
460 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
461 ret = -EINVAL;
462 goto out;
463 }
464
465 assignment = kstrdup(str, GFP_KERNEL);
466 if (!assignment) {
467 ret = -ENOMEM;
468 goto out;
469 }
470
471 attrs->assignment_str[attrs->n_assignments++] = assignment;
472 }
9b1ae035
TZ		 473 out:
474 return ret;
475}
476
7ef224d1
TZ		 477static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
478{
479 struct hist_trigger_attrs *attrs;
480 int ret = 0;
481
482 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
483 if (!attrs)
484 return ERR_PTR(-ENOMEM);
485
486 while (trigger_str) {
487 char *str = strsep(&trigger_str, ":");
488
9b1ae035
TZ		 489 if (strchr(str, '=')) {
490 ret = parse_assignment(str, attrs);
491 if (ret)
492 goto free;
493 } else if (strcmp(str, "pause") == 0)
83e99914
TZ		 494 attrs->pause = true;
495 else if ((strcmp(str, "cont") == 0) ||
496 (strcmp(str, "continue") == 0))
497 attrs->cont = true;
e86ae9ba
TZ		 498 else if (strcmp(str, "clear") == 0)
499 attrs->clear = true;
9b1ae035 500 else {
7ef224d1
TZ		 501 ret = -EINVAL;
502 goto free;
503 }
504 }
505
506 if (!attrs->keys_str) {
507 ret = -EINVAL;
508 goto free;
509 }
510
511 return attrs;
512 free:
513 destroy_hist_trigger_attrs(attrs);
514
515 return ERR_PTR(ret);
516}
517
6b4827ad
TZ		 518static inline void save_comm(char *comm, struct task_struct *task)
519{
520 if (!task->pid) {
521 strcpy(comm, "<idle>");
522 return;
523 }
524
525 if (WARN_ON_ONCE(task->pid < 0)) {
526 strcpy(comm, "<XXX>");
527 return;
528 }
529
530 memcpy(comm, task->comm, TASK_COMM_LEN);
531}
532
af6a29bc
TZ		 533static void hist_elt_data_free(struct hist_elt_data *elt_data)
534{
535 kfree(elt_data->comm);
536 kfree(elt_data);
537}
538
539static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
6b4827ad 540{
af6a29bc
TZ		 541 struct hist_elt_data *elt_data = elt->private_data;
542
543 hist_elt_data_free(elt_data);
6b4827ad
TZ		 544}
545
af6a29bc 546static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
6b4827ad
TZ		 547{
548 struct hist_trigger_data *hist_data = elt->map->private_data;
af6a29bc
TZ		 549 unsigned int size = TASK_COMM_LEN;
550 struct hist_elt_data *elt_data;
6b4827ad
TZ		 551 struct hist_field *key_field;
552 unsigned int i;
553
af6a29bc
TZ		 554 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
555 if (!elt_data)
556 return -ENOMEM;
557
6b4827ad
TZ		 558 for_each_hist_key_field(i, hist_data) {
559 key_field = hist_data->fields[i];
560
561 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
af6a29bc
TZ		 562 elt_data->comm = kzalloc(size, GFP_KERNEL);
563 if (!elt_data->comm) {
564 kfree(elt_data);
6b4827ad 565 return -ENOMEM;
af6a29bc 566 }
6b4827ad
TZ		 567 break;
568 }
569 }
570
af6a29bc
TZ		 571 elt->private_data = elt_data;
572
6b4827ad
TZ		 573 return 0;
574}
575
af6a29bc 576static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
6b4827ad 577{
af6a29bc 578 struct hist_elt_data *elt_data = elt->private_data;
6b4827ad 579
af6a29bc
TZ		 580 if (elt_data->comm)
581 save_comm(elt_data->comm, current);
6b4827ad
TZ		 582}
583
af6a29bc
TZ		 584static const struct tracing_map_ops hist_trigger_elt_data_ops = {
585 .elt_alloc = hist_trigger_elt_data_alloc,
586 .elt_free = hist_trigger_elt_data_free,
587 .elt_init = hist_trigger_elt_data_init,
6b4827ad
TZ		 588};
589
2ece94fb
TZ		 590static const char *get_hist_field_flags(struct hist_field *hist_field)
591{
592 const char *flags_str = NULL;
593
594 if (hist_field->flags & HIST_FIELD_FL_HEX)
595 flags_str = "hex";
596 else if (hist_field->flags & HIST_FIELD_FL_SYM)
597 flags_str = "sym";
598 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
599 flags_str = "sym-offset";
600 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
601 flags_str = "execname";
602 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
603 flags_str = "syscall";
604 else if (hist_field->flags & HIST_FIELD_FL_LOG2)
605 flags_str = "log2";
606 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
607 flags_str = "usecs";
608
609 return flags_str;
610}
611
100719dc
TZ		 612static void expr_field_str(struct hist_field *field, char *expr)
613{
614 strcat(expr, hist_field_name(field, 0));
615
616 if (field->flags) {
617 const char *flags_str = get_hist_field_flags(field);
618
619 if (flags_str) {
620 strcat(expr, ".");
621 strcat(expr, flags_str);
622 }
623 }
624}
625
626static char *expr_str(struct hist_field *field, unsigned int level)
627{
628 char *expr;
629
630 if (level > 1)
631 return NULL;
632
633 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
634 if (!expr)
635 return NULL;
636
637 if (!field->operands[0]) {
638 expr_field_str(field, expr);
639 return expr;
640 }
641
642 if (field->operator == FIELD_OP_UNARY_MINUS) {
643 char *subexpr;
644
645 strcat(expr, "-(");
646 subexpr = expr_str(field->operands[0], ++level);
647 if (!subexpr) {
648 kfree(expr);
649 return NULL;
650 }
651 strcat(expr, subexpr);
652 strcat(expr, ")");
653
654 kfree(subexpr);
655
656 return expr;
657 }
658
659 expr_field_str(field->operands[0], expr);
660
661 switch (field->operator) {
662 case FIELD_OP_MINUS:
663 strcat(expr, "-");
664 break;
665 case FIELD_OP_PLUS:
666 strcat(expr, "+");
667 break;
668 default:
669 kfree(expr);
670 return NULL;
671 }
672
673 expr_field_str(field->operands[1], expr);
674
675 return expr;
676}
677
678static int contains_operator(char *str)
679{
680 enum field_op_id field_op = FIELD_OP_NONE;
681 char *op;
682
683 op = strpbrk(str, "+-");
684 if (!op)
685 return FIELD_OP_NONE;
686
687 switch (*op) {
688 case '-':
689 if (*str == '-')
690 field_op = FIELD_OP_UNARY_MINUS;
691 else
692 field_op = FIELD_OP_MINUS;
693 break;
694 case '+':
695 field_op = FIELD_OP_PLUS;
696 break;
697 default:
698 break;
699 }
700
701 return field_op;
702}
703
5819eadd
TZ		 704static void destroy_hist_field(struct hist_field *hist_field,
705 unsigned int level)
7ef224d1 706{
5819eadd
TZ		 707 unsigned int i;
708
100719dc 709 if (level > 3)
5819eadd
TZ		 710 return;
711
712 if (!hist_field)
713 return;
714
715 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
716 destroy_hist_field(hist_field->operands[i], level + 1);
717
30350d65 718 kfree(hist_field->var.name);
100719dc 719 kfree(hist_field->name);
30350d65 720
7ef224d1
TZ		 721 kfree(hist_field);
722}
723
b559d003
TZ		 724static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
725 struct ftrace_event_field *field,
30350d65
TZ		 726 unsigned long flags,
727 char *var_name)
7ef224d1
TZ		 728{
729 struct hist_field *hist_field;
730
731 if (field && is_function_field(field))
732 return NULL;
733
734 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
735 if (!hist_field)
736 return NULL;
737
b559d003
TZ		 738 hist_field->hist_data = hist_data;
739
100719dc
TZ		 740 if (flags & HIST_FIELD_FL_EXPR)
741 goto out; /* caller will populate */
742
7ef224d1
TZ		 743 if (flags & HIST_FIELD_FL_HITCOUNT) {
744 hist_field->fn = hist_field_counter;
745 goto out;
746 }
747
69a0200c
TZ		 748 if (flags & HIST_FIELD_FL_STACKTRACE) {
749 hist_field->fn = hist_field_none;
750 goto out;
751 }
752
4b94f5b7 753 if (flags & HIST_FIELD_FL_LOG2) {
5819eadd 754 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
4b94f5b7 755 hist_field->fn = hist_field_log2;
30350d65 756 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
5819eadd 757 hist_field->size = hist_field->operands[0]->size;
4b94f5b7
NK		 758 goto out;
759 }
760
ad42febe
TZ		 761 if (flags & HIST_FIELD_FL_TIMESTAMP) {
762 hist_field->fn = hist_field_timestamp;
763 hist_field->size = sizeof(u64);
764 goto out;
765 }
766
432480c5
TZ		 767 if (WARN_ON_ONCE(!field))
768 goto out;
769
7ef224d1
TZ		 770 if (is_string_field(field)) {
771 flags |= HIST_FIELD_FL_STRING;
79e577cb
NK		 772
773 if (field->filter_type == FILTER_STATIC_STRING)
774 hist_field->fn = hist_field_string;
775 else if (field->filter_type == FILTER_DYN_STRING)
776 hist_field->fn = hist_field_dynstring;
777 else
778 hist_field->fn = hist_field_pstring;
7ef224d1
TZ		 779 } else {
780 hist_field->fn = select_value_fn(field->size,
781 field->is_signed);
782 if (!hist_field->fn) {
5819eadd 783 destroy_hist_field(hist_field, 0);
7ef224d1
TZ		 784 return NULL;
785 }
786 }
787 out:
788 hist_field->field = field;
789 hist_field->flags = flags;
790
30350d65
TZ		 791 if (var_name) {
792 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
793 if (!hist_field->var.name)
794 goto free;
795 }
796
7ef224d1 797 return hist_field;
30350d65
TZ		 798 free:
799 destroy_hist_field(hist_field, 0);
800 return NULL;
7ef224d1
TZ		 801}
802
803static void destroy_hist_fields(struct hist_trigger_data *hist_data)
804{
805 unsigned int i;
806
30350d65 807 for (i = 0; i < HIST_FIELDS_MAX; i++) {
7ef224d1 808 if (hist_data->fields[i]) {
5819eadd 809 destroy_hist_field(hist_data->fields[i], 0);
7ef224d1
TZ		 810 hist_data->fields[i] = NULL;
811 }
812 }
813}
814
100719dc
TZ		 815static struct ftrace_event_field *
816parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
817 char *field_str, unsigned long *flags)
818{
819 struct ftrace_event_field *field = NULL;
820 char *field_name, *modifier, *str;
821
822 modifier = str = kstrdup(field_str, GFP_KERNEL);
823 if (!modifier)
824 return ERR_PTR(-ENOMEM);
825
826 field_name = strsep(&modifier, ".");
827 if (modifier) {
828 if (strcmp(modifier, "hex") == 0)
829 *flags |= HIST_FIELD_FL_HEX;
830 else if (strcmp(modifier, "sym") == 0)
831 *flags |= HIST_FIELD_FL_SYM;
832 else if (strcmp(modifier, "sym-offset") == 0)
833 *flags |= HIST_FIELD_FL_SYM_OFFSET;
834 else if ((strcmp(modifier, "execname") == 0) &&
835 (strcmp(field_name, "common_pid") == 0))
836 *flags |= HIST_FIELD_FL_EXECNAME;
837 else if (strcmp(modifier, "syscall") == 0)
838 *flags |= HIST_FIELD_FL_SYSCALL;
839 else if (strcmp(modifier, "log2") == 0)
840 *flags |= HIST_FIELD_FL_LOG2;
841 else if (strcmp(modifier, "usecs") == 0)
842 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
843 else {
844 field = ERR_PTR(-EINVAL);
845 goto out;
846 }
847 }
848
849 if (strcmp(field_name, "common_timestamp") == 0) {
850 *flags |= HIST_FIELD_FL_TIMESTAMP;
851 hist_data->enable_timestamps = true;
852 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
853 hist_data->attrs->ts_in_usecs = true;
854 } else {
855 field = trace_find_event_field(file->event_call, field_name);
856 if (!field || !field->size) {
857 field = ERR_PTR(-EINVAL);
858 goto out;
859 }
860 }
861 out:
862 kfree(str);
863
864 return field;
865}
866
867static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
868 struct trace_event_file *file, char *str,
869 unsigned long *flags, char *var_name)
870{
871 struct ftrace_event_field *field = NULL;
872 struct hist_field *hist_field = NULL;
873 int ret = 0;
874
875 field = parse_field(hist_data, file, str, flags);
876 if (IS_ERR(field)) {
877 ret = PTR_ERR(field);
878 goto out;
879 }
880
881 hist_field = create_hist_field(hist_data, field, *flags, var_name);
882 if (!hist_field) {
883 ret = -ENOMEM;
884 goto out;
885 }
886
887 return hist_field;
888 out:
889 return ERR_PTR(ret);
890}
891
892static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
893 struct trace_event_file *file,
894 char *str, unsigned long flags,
895 char *var_name, unsigned int level);
896
897static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
898 struct trace_event_file *file,
899 char *str, unsigned long flags,
900 char *var_name, unsigned int level)
901{
902 struct hist_field *operand1, *expr = NULL;
903 unsigned long operand_flags;
904 int ret = 0;
905 char *s;
906
907 /* we support only -(xxx) i.e. explicit parens required */
908
909 if (level > 3) {
910 ret = -EINVAL;
911 goto free;
912 }
913
914 str++; /* skip leading '-' */
915
916 s = strchr(str, '(');
917 if (s)
918 str++;
919 else {
920 ret = -EINVAL;
921 goto free;
922 }
923
924 s = strrchr(str, ')');
925 if (s)
926 *s = '\0';
927 else {
928 ret = -EINVAL; /* no closing ')' */
929 goto free;
930 }
931
932 flags |= HIST_FIELD_FL_EXPR;
933 expr = create_hist_field(hist_data, NULL, flags, var_name);
934 if (!expr) {
935 ret = -ENOMEM;
936 goto free;
937 }
938
939 operand_flags = 0;
940 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
941 if (IS_ERR(operand1)) {
942 ret = PTR_ERR(operand1);
943 goto free;
944 }
945
946 expr->flags |= operand1->flags &
947 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
948 expr->fn = hist_field_unary_minus;
949 expr->operands[0] = operand1;
950 expr->operator = FIELD_OP_UNARY_MINUS;
951 expr->name = expr_str(expr, 0);
952
953 return expr;
954 free:
955 destroy_hist_field(expr, 0);
956 return ERR_PTR(ret);
957}
958
959static int check_expr_operands(struct hist_field *operand1,
960 struct hist_field *operand2)
961{
962 unsigned long operand1_flags = operand1->flags;
963 unsigned long operand2_flags = operand2->flags;
964
965 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
966 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS))
967 return -EINVAL;
968
969 return 0;
970}
971
972static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
973 struct trace_event_file *file,
974 char *str, unsigned long flags,
975 char *var_name, unsigned int level)
976{
977 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
978 unsigned long operand_flags;
979 int field_op, ret = -EINVAL;
980 char *sep, *operand1_str;
981
982 if (level > 3)
983 return ERR_PTR(-EINVAL);
984
985 field_op = contains_operator(str);
986
987 if (field_op == FIELD_OP_NONE)
988 return parse_atom(hist_data, file, str, &flags, var_name);
989
990 if (field_op == FIELD_OP_UNARY_MINUS)
991 return parse_unary(hist_data, file, str, flags, var_name, ++level);
992
993 switch (field_op) {
994 case FIELD_OP_MINUS:
995 sep = "-";
996 break;
997 case FIELD_OP_PLUS:
998 sep = "+";
999 break;
1000 default:
1001 goto free;
1002 }
1003
1004 operand1_str = strsep(&str, sep);
1005 if (!operand1_str || !str)
1006 goto free;
1007
1008 operand_flags = 0;
1009 operand1 = parse_atom(hist_data, file, operand1_str,
1010 &operand_flags, NULL);
1011 if (IS_ERR(operand1)) {
1012 ret = PTR_ERR(operand1);
1013 operand1 = NULL;
1014 goto free;
1015 }
1016
1017 /* rest of string could be another expression e.g. b+c in a+b+c */
1018 operand_flags = 0;
1019 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
1020 if (IS_ERR(operand2)) {
1021 ret = PTR_ERR(operand2);
1022 operand2 = NULL;
1023 goto free;
1024 }
1025
1026 ret = check_expr_operands(operand1, operand2);
1027 if (ret)
1028 goto free;
1029
1030 flags |= HIST_FIELD_FL_EXPR;
1031
1032 flags |= operand1->flags &
1033 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1034
1035 expr = create_hist_field(hist_data, NULL, flags, var_name);
1036 if (!expr) {
1037 ret = -ENOMEM;
1038 goto free;
1039 }
1040
1041 expr->operands[0] = operand1;
1042 expr->operands[1] = operand2;
1043 expr->operator = field_op;
1044 expr->name = expr_str(expr, 0);
1045
1046 switch (field_op) {
1047 case FIELD_OP_MINUS:
1048 expr->fn = hist_field_minus;
1049 break;
1050 case FIELD_OP_PLUS:
1051 expr->fn = hist_field_plus;
1052 break;
1053 default:
1054 goto free;
1055 }
1056
1057 return expr;
1058 free:
1059 destroy_hist_field(operand1, 0);
1060 destroy_hist_field(operand2, 0);
1061 destroy_hist_field(expr, 0);
1062
1063 return ERR_PTR(ret);
1064}
1065
7ef224d1
TZ		 1066static int create_hitcount_val(struct hist_trigger_data *hist_data)
1067{
1068 hist_data->fields[HITCOUNT_IDX] =
30350d65 1069 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
7ef224d1
TZ		 1070 if (!hist_data->fields[HITCOUNT_IDX])
1071 return -ENOMEM;
1072
1073 hist_data->n_vals++;
30350d65 1074 hist_data->n_fields++;
7ef224d1
TZ		 1075
1076 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
1077 return -EINVAL;
1078
1079 return 0;
1080}
1081
30350d65
TZ		 1082static int __create_val_field(struct hist_trigger_data *hist_data,
1083 unsigned int val_idx,
1084 struct trace_event_file *file,
1085 char *var_name, char *field_str,
1086 unsigned long flags)
f2606835 1087{
100719dc 1088 struct hist_field *hist_field;
f2606835
TZ		 1089 int ret = 0;
1090
100719dc
TZ		 1091 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
1092 if (IS_ERR(hist_field)) {
1093 ret = PTR_ERR(hist_field);
f2606835
TZ		 1094 goto out;
1095 }
1096
100719dc
TZ		 1097 hist_data->fields[val_idx] = hist_field;
1098
f2606835 1099 ++hist_data->n_vals;
30350d65 1100 ++hist_data->n_fields;
f2606835 1101
30350d65 1102 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
f2606835
TZ		 1103 ret = -EINVAL;
1104 out:
1105 return ret;
1106}
1107
30350d65
TZ		 1108static int create_val_field(struct hist_trigger_data *hist_data,
1109 unsigned int val_idx,
1110 struct trace_event_file *file,
1111 char *field_str)
1112{
1113 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
1114 return -EINVAL;
1115
1116 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
1117}
1118
1119static int create_var_field(struct hist_trigger_data *hist_data,
1120 unsigned int val_idx,
1121 struct trace_event_file *file,
1122 char *var_name, char *expr_str)
1123{
1124 unsigned long flags = 0;
1125
1126 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
1127 return -EINVAL;
1128 if (find_var(hist_data, file, var_name) && !hist_data->remove) {
1129 return -EINVAL;
1130 }
1131
1132 flags |= HIST_FIELD_FL_VAR;
1133 hist_data->n_vars++;
1134 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
1135 return -EINVAL;
1136
1137 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
1138}
1139
7ef224d1
TZ		 1140static int create_val_fields(struct hist_trigger_data *hist_data,
1141 struct trace_event_file *file)
1142{
f2606835 1143 char *fields_str, *field_str;
30350d65 1144 unsigned int i, j = 1;
7ef224d1
TZ		 1145 int ret;
1146
1147 ret = create_hitcount_val(hist_data);
f2606835
TZ		 1148 if (ret)
1149 goto out;
7ef224d1 1150
f2606835
TZ		 1151 fields_str = hist_data->attrs->vals_str;
1152 if (!fields_str)
1153 goto out;
1154
1155 strsep(&fields_str, "=");
1156 if (!fields_str)
1157 goto out;
1158
1159 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
1160 j < TRACING_MAP_VALS_MAX; i++) {
1161 field_str = strsep(&fields_str, ",");
1162 if (!field_str)
1163 break;
30350d65 1164
f2606835
TZ		 1165 if (strcmp(field_str, "hitcount") == 0)
1166 continue;
30350d65 1167
f2606835
TZ		 1168 ret = create_val_field(hist_data, j++, file, field_str);
1169 if (ret)
1170 goto out;
1171 }
30350d65 1172
f2606835
TZ		 1173 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
1174 ret = -EINVAL;
1175 out:
7ef224d1
TZ		 1176 return ret;
1177}
1178
1179static int create_key_field(struct hist_trigger_data *hist_data,
1180 unsigned int key_idx,
76a3b0c8 1181 unsigned int key_offset,
7ef224d1
TZ		 1182 struct trace_event_file *file,
1183 char *field_str)
1184{
30350d65 1185 struct hist_field *hist_field = NULL;
100719dc 1186
7ef224d1
TZ		 1187 unsigned long flags = 0;
1188 unsigned int key_size;
1189 int ret = 0;
1190
30350d65 1191 if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
7ef224d1
TZ		 1192 return -EINVAL;
1193
1194 flags |= HIST_FIELD_FL_KEY;
1195
69a0200c
TZ		 1196 if (strcmp(field_str, "stacktrace") == 0) {
1197 flags |= HIST_FIELD_FL_STACKTRACE;
1198 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
30350d65 1199 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
69a0200c 1200 } else {
100719dc
TZ		 1201 hist_field = parse_expr(hist_data, file, field_str, flags,
1202 NULL, 0);
1203 if (IS_ERR(hist_field)) {
1204 ret = PTR_ERR(hist_field);
1205 goto out;
69a0200c
TZ		 1206 }
1207
100719dc 1208 key_size = hist_field->size;
7ef224d1
TZ		 1209 }
1210
100719dc 1211 hist_data->fields[key_idx] = hist_field;
7ef224d1
TZ		 1212
1213 key_size = ALIGN(key_size, sizeof(u64));
1214 hist_data->fields[key_idx]->size = key_size;
76a3b0c8 1215 hist_data->fields[key_idx]->offset = key_offset;
100719dc 1216
76a3b0c8 1217 hist_data->key_size += key_size;
100719dc 1218
7ef224d1
TZ		 1219 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
1220 ret = -EINVAL;
1221 goto out;
1222 }
1223
1224 hist_data->n_keys++;
30350d65 1225 hist_data->n_fields++;
7ef224d1
TZ		 1226
1227 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
1228 return -EINVAL;
1229
1230 ret = key_size;
1231 out:
1232 return ret;
1233}
1234
1235static int create_key_fields(struct hist_trigger_data *hist_data,
1236 struct trace_event_file *file)
1237{
76a3b0c8 1238 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
7ef224d1
TZ		 1239 char *fields_str, *field_str;
1240 int ret = -EINVAL;
1241
1242 fields_str = hist_data->attrs->keys_str;
1243 if (!fields_str)
1244 goto out;
1245
1246 strsep(&fields_str, "=");
1247 if (!fields_str)
1248 goto out;
1249
76a3b0c8 1250 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
7ef224d1
TZ		 1251 field_str = strsep(&fields_str, ",");
1252 if (!field_str)
1253 break;
76a3b0c8
TZ		 1254 ret = create_key_field(hist_data, i, key_offset,
1255 file, field_str);
7ef224d1
TZ		 1256 if (ret < 0)
1257 goto out;
76a3b0c8 1258 key_offset += ret;
7ef224d1
TZ		 1259 }
1260 if (fields_str) {
1261 ret = -EINVAL;
1262 goto out;
1263 }
1264 ret = 0;
1265 out:
1266 return ret;
1267}
1268
30350d65
TZ		 1269static int create_var_fields(struct hist_trigger_data *hist_data,
1270 struct trace_event_file *file)
1271{
1272 unsigned int i, j = hist_data->n_vals;
1273 int ret = 0;
1274
1275 unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
1276
1277 for (i = 0; i < n_vars; i++) {
1278 char *var_name = hist_data->attrs->var_defs.name[i];
1279 char *expr = hist_data->attrs->var_defs.expr[i];
1280
1281 ret = create_var_field(hist_data, j++, file, var_name, expr);
1282 if (ret)
1283 goto out;
1284 }
1285 out:
1286 return ret;
1287}
1288
1289static void free_var_defs(struct hist_trigger_data *hist_data)
1290{
1291 unsigned int i;
1292
1293 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1294 kfree(hist_data->attrs->var_defs.name[i]);
1295 kfree(hist_data->attrs->var_defs.expr[i]);
1296 }
1297
1298 hist_data->attrs->var_defs.n_vars = 0;
1299}
1300
1301static int parse_var_defs(struct hist_trigger_data *hist_data)
1302{
1303 char *s, *str, *var_name, *field_str;
1304 unsigned int i, j, n_vars = 0;
1305 int ret = 0;
1306
1307 for (i = 0; i < hist_data->attrs->n_assignments; i++) {
1308 str = hist_data->attrs->assignment_str[i];
1309 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
1310 field_str = strsep(&str, ",");
1311 if (!field_str)
1312 break;
1313
1314 var_name = strsep(&field_str, "=");
1315 if (!var_name || !field_str) {
1316 ret = -EINVAL;
1317 goto free;
1318 }
1319
1320 if (n_vars == TRACING_MAP_VARS_MAX) {
1321 ret = -EINVAL;
1322 goto free;
1323 }
1324
1325 s = kstrdup(var_name, GFP_KERNEL);
1326 if (!s) {
1327 ret = -ENOMEM;
1328 goto free;
1329 }
1330 hist_data->attrs->var_defs.name[n_vars] = s;
1331
1332 s = kstrdup(field_str, GFP_KERNEL);
1333 if (!s) {
1334 kfree(hist_data->attrs->var_defs.name[n_vars]);
1335 ret = -ENOMEM;
1336 goto free;
1337 }
1338 hist_data->attrs->var_defs.expr[n_vars++] = s;
1339
1340 hist_data->attrs->var_defs.n_vars = n_vars;
1341 }
1342 }
1343
1344 return ret;
1345 free:
1346 free_var_defs(hist_data);
1347
1348 return ret;
1349}
1350
7ef224d1
TZ		 1351static int create_hist_fields(struct hist_trigger_data *hist_data,
1352 struct trace_event_file *file)
1353{
1354 int ret;
1355
30350d65
TZ		 1356 ret = parse_var_defs(hist_data);
1357 if (ret)
1358 goto out;
1359
7ef224d1
TZ		 1360 ret = create_val_fields(hist_data, file);
1361 if (ret)
1362 goto out;
1363
30350d65 1364 ret = create_var_fields(hist_data, file);
7ef224d1
TZ		 1365 if (ret)
1366 goto out;
1367
30350d65
TZ		 1368 ret = create_key_fields(hist_data, file);
1369 if (ret)
1370 goto out;
7ef224d1 1371 out:
30350d65
TZ		 1372 free_var_defs(hist_data);
1373
7ef224d1
TZ		 1374 return ret;
1375}
1376
e62347d2
TZ		 1377static int is_descending(const char *str)
1378{
1379 if (!str)
1380 return 0;
1381
1382 if (strcmp(str, "descending") == 0)
1383 return 1;
1384
1385 if (strcmp(str, "ascending") == 0)
1386 return 0;
1387
1388 return -EINVAL;
1389}
1390
7ef224d1
TZ		 1391static int create_sort_keys(struct hist_trigger_data *hist_data)
1392{
e62347d2 1393 char *fields_str = hist_data->attrs->sort_key_str;
e62347d2
TZ		 1394 struct tracing_map_sort_key *sort_key;
1395 int descending, ret = 0;
30350d65 1396 unsigned int i, j, k;
e62347d2
TZ		 1397
1398 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
1399
1400 if (!fields_str)
1401 goto out;
1402
1403 strsep(&fields_str, "=");
1404 if (!fields_str) {
1405 ret = -EINVAL;
1406 goto out;
1407 }
1408
1409 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
85013256 1410 struct hist_field *hist_field;
e62347d2 1411 char *field_str, *field_name;
85013256 1412 const char *test_name;
e62347d2
TZ		 1413
1414 sort_key = &hist_data->sort_keys[i];
1415
1416 field_str = strsep(&fields_str, ",");
1417 if (!field_str) {
1418 if (i == 0)
1419 ret = -EINVAL;
1420 break;
1421 }
1422
1423 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
1424 ret = -EINVAL;
1425 break;
1426 }
7ef224d1 1427
e62347d2
TZ		 1428 field_name = strsep(&field_str, ".");
1429 if (!field_name) {
1430 ret = -EINVAL;
1431 break;
1432 }
1433
1434 if (strcmp(field_name, "hitcount") == 0) {
1435 descending = is_descending(field_str);
1436 if (descending < 0) {
1437 ret = descending;
1438 break;
1439 }
1440 sort_key->descending = descending;
1441 continue;
1442 }
7ef224d1 1443
30350d65
TZ		 1444 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
1445 unsigned int idx;
1446
85013256 1447 hist_field = hist_data->fields[j];
30350d65
TZ		 1448 if (hist_field->flags & HIST_FIELD_FL_VAR)
1449 continue;
1450
1451 idx = k++;
1452
85013256
TZ		 1453 test_name = hist_field_name(hist_field, 0);
1454
1455 if (strcmp(field_name, test_name) == 0) {
30350d65 1456 sort_key->field_idx = idx;
e62347d2
TZ		 1457 descending = is_descending(field_str);
1458 if (descending < 0) {
1459 ret = descending;
1460 goto out;
1461 }
1462 sort_key->descending = descending;
1463 break;
1464 }
1465 }
1466 if (j == hist_data->n_fields) {
1467 ret = -EINVAL;
1468 break;
1469 }
1470 }
30350d65 1471
e62347d2
TZ		 1472 hist_data->n_sort_keys = i;
1473 out:
7ef224d1
TZ		 1474 return ret;
1475}
1476
1477static void destroy_hist_data(struct hist_trigger_data *hist_data)
1478{
1479 destroy_hist_trigger_attrs(hist_data->attrs);
1480 destroy_hist_fields(hist_data);
1481 tracing_map_destroy(hist_data->map);
1482 kfree(hist_data);
1483}
1484
1485static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
1486{
1487 struct tracing_map *map = hist_data->map;
1488 struct ftrace_event_field *field;
1489 struct hist_field *hist_field;
d50c744e 1490 int i, idx;
7ef224d1
TZ		 1491
1492 for_each_hist_field(i, hist_data) {
1493 hist_field = hist_data->fields[i];
1494 if (hist_field->flags & HIST_FIELD_FL_KEY) {
1495 tracing_map_cmp_fn_t cmp_fn;
1496
1497 field = hist_field->field;
1498
69a0200c
TZ		 1499 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1500 cmp_fn = tracing_map_cmp_none;
ad42febe
TZ		 1501 else if (!field)
1502 cmp_fn = tracing_map_cmp_num(hist_field->size,
1503 hist_field->is_signed);
69a0200c 1504 else if (is_string_field(field))
7ef224d1
TZ		 1505 cmp_fn = tracing_map_cmp_string;
1506 else
1507 cmp_fn = tracing_map_cmp_num(field->size,
1508 field->is_signed);
76a3b0c8
TZ		 1509 idx = tracing_map_add_key_field(map,
1510 hist_field->offset,
1511 cmp_fn);
30350d65 1512 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
7ef224d1
TZ		 1513 idx = tracing_map_add_sum_field(map);
1514
1515 if (idx < 0)
1516 return idx;
30350d65
TZ		 1517
1518 if (hist_field->flags & HIST_FIELD_FL_VAR) {
1519 idx = tracing_map_add_var(map);
1520 if (idx < 0)
1521 return idx;
1522 hist_field->var.idx = idx;
1523 hist_field->var.hist_data = hist_data;
1524 }
7ef224d1
TZ		 1525 }
1526
1527 return 0;
1528}
1529
1530static struct hist_trigger_data *
1531create_hist_data(unsigned int map_bits,
1532 struct hist_trigger_attrs *attrs,
30350d65
TZ		 1533 struct trace_event_file *file,
1534 bool remove)
7ef224d1 1535{
6b4827ad 1536 const struct tracing_map_ops *map_ops = NULL;
7ef224d1
TZ		 1537 struct hist_trigger_data *hist_data;
1538 int ret = 0;
1539
1540 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
1541 if (!hist_data)
1542 return ERR_PTR(-ENOMEM);
1543
1544 hist_data->attrs = attrs;
30350d65 1545 hist_data->remove = remove;
7ef224d1
TZ		 1546
1547 ret = create_hist_fields(hist_data, file);
1548 if (ret)
1549 goto free;
1550
1551 ret = create_sort_keys(hist_data);
1552 if (ret)
1553 goto free;
1554
af6a29bc 1555 map_ops = &hist_trigger_elt_data_ops;
6b4827ad 1556
7ef224d1 1557 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
6b4827ad 1558 map_ops, hist_data);
7ef224d1
TZ		 1559 if (IS_ERR(hist_data->map)) {
1560 ret = PTR_ERR(hist_data->map);
1561 hist_data->map = NULL;
1562 goto free;
1563 }
1564
1565 ret = create_tracing_map_fields(hist_data);
1566 if (ret)
1567 goto free;
1568
1569 ret = tracing_map_init(hist_data->map);
1570 if (ret)
1571 goto free;
1572
1573 hist_data->event_file = file;
1574 out:
1575 return hist_data;
1576 free:
1577 hist_data->attrs = NULL;
1578
1579 destroy_hist_data(hist_data);
1580
1581 hist_data = ERR_PTR(ret);
1582
1583 goto out;
1584}
1585
1586static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
fbd302cb
TZ		 1587 struct tracing_map_elt *elt, void *rec,
1588 struct ring_buffer_event *rbe)
7ef224d1
TZ		 1589{
1590 struct hist_field *hist_field;
30350d65 1591 unsigned int i, var_idx;
7ef224d1
TZ		 1592 u64 hist_val;
1593
1594 for_each_hist_val_field(i, hist_data) {
1595 hist_field = hist_data->fields[i];
df35d93b 1596 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
30350d65
TZ		 1597 if (hist_field->flags & HIST_FIELD_FL_VAR) {
1598 var_idx = hist_field->var.idx;
1599 tracing_map_set_var(elt, var_idx, hist_val);
1600 continue;
1601 }
7ef224d1
TZ		 1602 tracing_map_update_sum(elt, i, hist_val);
1603 }
30350d65
TZ		 1604
1605 for_each_hist_key_field(i, hist_data) {
1606 hist_field = hist_data->fields[i];
1607 if (hist_field->flags & HIST_FIELD_FL_VAR) {
df35d93b 1608 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
30350d65
TZ		 1609 var_idx = hist_field->var.idx;
1610 tracing_map_set_var(elt, var_idx, hist_val);
1611 }
1612 }
7ef224d1
TZ		 1613}
1614
6a475cb1
TZ		 1615static inline void add_to_key(char *compound_key, void *key,
1616 struct hist_field *key_field, void *rec)
1617{
1618 size_t size = key_field->size;
1619
1620 if (key_field->flags & HIST_FIELD_FL_STRING) {
1621 struct ftrace_event_field *field;
1622
1623 field = key_field->field;
1624 if (field->filter_type == FILTER_DYN_STRING)
1625 size = *(u32 *)(rec + field->offset) >> 16;
1626 else if (field->filter_type == FILTER_PTR_STRING)
1627 size = strlen(key);
1628 else if (field->filter_type == FILTER_STATIC_STRING)
1629 size = field->size;
1630
1631 /* ensure NULL-termination */
1632 if (size > key_field->size - 1)
1633 size = key_field->size - 1;
1634 }
1635
1636 memcpy(compound_key + key_field->offset, key, size);
1637}
1638
1ac4f51c 1639static void event_hist_trigger(struct event_trigger_data *data, void *rec,
fbd302cb 1640 struct ring_buffer_event *rbe)
7ef224d1
TZ		 1641{
1642 struct hist_trigger_data *hist_data = data->private_data;
6a475cb1 1643 bool use_compound_key = (hist_data->n_keys > 1);
69a0200c 1644 unsigned long entries[HIST_STACKTRACE_DEPTH];
76a3b0c8 1645 char compound_key[HIST_KEY_SIZE_MAX];
df35d93b 1646 struct tracing_map_elt *elt = NULL;
69a0200c 1647 struct stack_trace stacktrace;
7ef224d1 1648 struct hist_field *key_field;
7ef224d1
TZ		 1649 u64 field_contents;
1650 void *key = NULL;
1651 unsigned int i;
1652
6a475cb1 1653 memset(compound_key, 0, hist_data->key_size);
76a3b0c8 1654
7ef224d1
TZ		 1655 for_each_hist_key_field(i, hist_data) {
1656 key_field = hist_data->fields[i];
1657
69a0200c
TZ		 1658 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
1659 stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
1660 stacktrace.entries = entries;
1661 stacktrace.nr_entries = 0;
1662 stacktrace.skip = HIST_STACKTRACE_SKIP;
76a3b0c8 1663
69a0200c
TZ		 1664 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
1665 save_stack_trace(&stacktrace);
1666
1667 key = entries;
1668 } else {
df35d93b 1669 field_contents = key_field->fn(key_field, elt, rbe, rec);
6a475cb1 1670 if (key_field->flags & HIST_FIELD_FL_STRING) {
69a0200c 1671 key = (void *)(unsigned long)field_contents;
6a475cb1
TZ		 1672 use_compound_key = true;
1673 } else
69a0200c 1674 key = (void *)&field_contents;
76a3b0c8 1675 }
6a475cb1
TZ		 1676
1677 if (use_compound_key)
1678 add_to_key(compound_key, key, key_field, rec);
7ef224d1
TZ		 1679 }
1680
6a475cb1 1681 if (use_compound_key)
76a3b0c8
TZ		 1682 key = compound_key;
1683
7ef224d1
TZ		 1684 elt = tracing_map_insert(hist_data->map, key);
1685 if (elt)
fbd302cb 1686 hist_trigger_elt_update(hist_data, elt, rec, rbe);
7ef224d1
TZ		 1687}
1688
69a0200c
TZ		 1689static void hist_trigger_stacktrace_print(struct seq_file *m,
1690 unsigned long *stacktrace_entries,
1691 unsigned int max_entries)
1692{
1693 char str[KSYM_SYMBOL_LEN];
1694 unsigned int spaces = 8;
1695 unsigned int i;
1696
1697 for (i = 0; i < max_entries; i++) {
1698 if (stacktrace_entries[i] == ULONG_MAX)
1699 return;
1700
1701 seq_printf(m, "%*c", 1 + spaces, ' ');
1702 sprint_symbol(str, stacktrace_entries[i]);
1703 seq_printf(m, "%s\n", str);
1704 }
1705}
1706
7ef224d1
TZ		 1707static void
1708hist_trigger_entry_print(struct seq_file *m,
1709 struct hist_trigger_data *hist_data, void *key,
1710 struct tracing_map_elt *elt)
1711{
1712 struct hist_field *key_field;
c6afad49 1713 char str[KSYM_SYMBOL_LEN];
69a0200c 1714 bool multiline = false;
85013256 1715 const char *field_name;
7ef224d1
TZ		 1716 unsigned int i;
1717 u64 uval;
1718
1719 seq_puts(m, "{ ");
1720
1721 for_each_hist_key_field(i, hist_data) {
1722 key_field = hist_data->fields[i];
1723
1724 if (i > hist_data->n_vals)
1725 seq_puts(m, ", ");
1726
85013256
TZ		 1727 field_name = hist_field_name(key_field, 0);
1728
0c4a6b46
TZ		 1729 if (key_field->flags & HIST_FIELD_FL_HEX) {
1730 uval = *(u64 *)(key + key_field->offset);
85013256 1731 seq_printf(m, "%s: %llx", field_name, uval);
c6afad49
TZ		 1732 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
1733 uval = *(u64 *)(key + key_field->offset);
1734 sprint_symbol_no_offset(str, uval);
85013256
TZ		 1735 seq_printf(m, "%s: [%llx] %-45s", field_name,
1736 uval, str);
c6afad49
TZ		 1737 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
1738 uval = *(u64 *)(key + key_field->offset);
1739 sprint_symbol(str, uval);
85013256
TZ		 1740 seq_printf(m, "%s: [%llx] %-55s", field_name,
1741 uval, str);
6b4827ad 1742 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
af6a29bc
TZ		 1743 struct hist_elt_data *elt_data = elt->private_data;
1744 char *comm;
1745
1746 if (WARN_ON_ONCE(!elt_data))
1747 return;
1748
1749 comm = elt_data->comm;
6b4827ad
TZ		 1750
1751 uval = *(u64 *)(key + key_field->offset);
85013256
TZ		 1752 seq_printf(m, "%s: %-16s[%10llu]", field_name,
1753 comm, uval);
31696198
TZ		 1754 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
1755 const char *syscall_name;
1756
1757 uval = *(u64 *)(key + key_field->offset);
1758 syscall_name = get_syscall_name(uval);
1759 if (!syscall_name)
1760 syscall_name = "unknown_syscall";
1761
85013256
TZ		 1762 seq_printf(m, "%s: %-30s[%3llu]", field_name,
1763 syscall_name, uval);
69a0200c
TZ		 1764 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
1765 seq_puts(m, "stacktrace:\n");
1766 hist_trigger_stacktrace_print(m,
1767 key + key_field->offset,
1768 HIST_STACKTRACE_DEPTH);
1769 multiline = true;
4b94f5b7 1770 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
85013256 1771 seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4b94f5b7 1772 *(u64 *)(key + key_field->offset));
0c4a6b46 1773 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
85013256 1774 seq_printf(m, "%s: %-50s", field_name,
76a3b0c8 1775 (char *)(key + key_field->offset));
7ef224d1 1776 } else {
76a3b0c8 1777 uval = *(u64 *)(key + key_field->offset);
85013256 1778 seq_printf(m, "%s: %10llu", field_name, uval);
7ef224d1
TZ		 1779 }
1780 }
1781
69a0200c
TZ		 1782 if (!multiline)
1783 seq_puts(m, " ");
1784
1785 seq_puts(m, "}");
7ef224d1
TZ		 1786
1787 seq_printf(m, " hitcount: %10llu",
1788 tracing_map_read_sum(elt, HITCOUNT_IDX));
1789
f2606835 1790 for (i = 1; i < hist_data->n_vals; i++) {
85013256
TZ		 1791 field_name = hist_field_name(hist_data->fields[i], 0);
1792
100719dc
TZ		 1793 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
1794 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
30350d65
TZ		 1795 continue;
1796
0c4a6b46 1797 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
85013256 1798 seq_printf(m, " %s: %10llx", field_name,
0c4a6b46
TZ		 1799 tracing_map_read_sum(elt, i));
1800 } else {
85013256 1801 seq_printf(m, " %s: %10llu", field_name,
0c4a6b46
TZ		 1802 tracing_map_read_sum(elt, i));
1803 }
f2606835
TZ		 1804 }
1805
7ef224d1
TZ		 1806 seq_puts(m, "\n");
1807}
1808
1809static int print_entries(struct seq_file *m,
1810 struct hist_trigger_data *hist_data)
1811{
1812 struct tracing_map_sort_entry **sort_entries = NULL;
1813 struct tracing_map *map = hist_data->map;
d50c744e 1814 int i, n_entries;
7ef224d1
TZ		 1815
1816 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
1817 hist_data->n_sort_keys,
1818 &sort_entries);
1819 if (n_entries < 0)
1820 return n_entries;
1821
1822 for (i = 0; i < n_entries; i++)
1823 hist_trigger_entry_print(m, hist_data,
1824 sort_entries[i]->key,
1825 sort_entries[i]->elt);
1826
1827 tracing_map_destroy_sort_entries(sort_entries, n_entries);
1828
1829 return n_entries;
1830}
1831
52a7f16d
TZ		 1832static void hist_trigger_show(struct seq_file *m,
1833 struct event_trigger_data *data, int n)
7ef224d1 1834{
7ef224d1 1835 struct hist_trigger_data *hist_data;
6e7a2398 1836 int n_entries;
7ef224d1 1837
52a7f16d
TZ		 1838 if (n > 0)
1839 seq_puts(m, "\n\n");
7ef224d1
TZ		 1840
1841 seq_puts(m, "# event histogram\n#\n# trigger info: ");
1842 data->ops->print(m, data->ops, data);
52a7f16d 1843 seq_puts(m, "#\n\n");
7ef224d1
TZ		 1844
1845 hist_data = data->private_data;
1846 n_entries = print_entries(m, hist_data);
6e7a2398 1847 if (n_entries < 0)
7ef224d1 1848 n_entries = 0;
7ef224d1
TZ		 1849
1850 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
1851 (u64)atomic64_read(&hist_data->map->hits),
1852 n_entries, (u64)atomic64_read(&hist_data->map->drops));
52a7f16d
TZ		 1853}
1854
1855static int hist_show(struct seq_file *m, void *v)
1856{
1857 struct event_trigger_data *data;
1858 struct trace_event_file *event_file;
1859 int n = 0, ret = 0;
1860
1861 mutex_lock(&event_mutex);
1862
1863 event_file = event_file_data(m->private);
1864 if (unlikely(!event_file)) {
1865 ret = -ENODEV;
1866 goto out_unlock;
1867 }
1868
1869 list_for_each_entry_rcu(data, &event_file->triggers, list) {
1870 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
1871 hist_trigger_show(m, data, n++);
1872 }
1873
7ef224d1
TZ		 1874 out_unlock:
1875 mutex_unlock(&event_mutex);
1876
1877 return ret;
1878}
1879
1880static int event_hist_open(struct inode *inode, struct file *file)
1881{
1882 return single_open(file, hist_show, file);
1883}
1884
1885const struct file_operations event_hist_fops = {
1886 .open = event_hist_open,
1887 .read = seq_read,
1888 .llseek = seq_lseek,
1889 .release = single_release,
1890};
1891
1892static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
1893{
85013256
TZ		 1894 const char *field_name = hist_field_name(hist_field, 0);
1895
30350d65
TZ		 1896 if (hist_field->var.name)
1897 seq_printf(m, "%s=", hist_field->var.name);
1898
ad42febe
TZ		 1899 if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
1900 seq_puts(m, "common_timestamp");
1901 else if (field_name)
1902 seq_printf(m, "%s", field_name);
1903
0c4a6b46
TZ		 1904 if (hist_field->flags) {
1905 const char *flags_str = get_hist_field_flags(hist_field);
1906
1907 if (flags_str)
1908 seq_printf(m, ".%s", flags_str);
1909 }
7ef224d1
TZ		 1910}
1911
1912static int event_hist_trigger_print(struct seq_file *m,
1913 struct event_trigger_ops *ops,
1914 struct event_trigger_data *data)
1915{
1916 struct hist_trigger_data *hist_data = data->private_data;
30350d65
TZ		 1917 struct hist_field *field;
1918 bool have_var = false;
7ef224d1
TZ		 1919 unsigned int i;
1920
5463bfda
TZ		 1921 seq_puts(m, "hist:");
1922
1923 if (data->name)
1924 seq_printf(m, "%s:", data->name);
1925
1926 seq_puts(m, "keys=");
7ef224d1
TZ		 1927
1928 for_each_hist_key_field(i, hist_data) {
30350d65 1929 field = hist_data->fields[i];
7ef224d1
TZ		 1930
1931 if (i > hist_data->n_vals)
1932 seq_puts(m, ",");
1933
30350d65 1934 if (field->flags & HIST_FIELD_FL_STACKTRACE)
69a0200c
TZ		 1935 seq_puts(m, "stacktrace");
1936 else
30350d65 1937 hist_field_print(m, field);
7ef224d1
TZ		 1938 }
1939
1940 seq_puts(m, ":vals=");
f2606835
TZ		 1941
1942 for_each_hist_val_field(i, hist_data) {
30350d65
TZ		 1943 field = hist_data->fields[i];
1944 if (field->flags & HIST_FIELD_FL_VAR) {
1945 have_var = true;
1946 continue;
1947 }
1948
f2606835
TZ		 1949 if (i == HITCOUNT_IDX)
1950 seq_puts(m, "hitcount");
1951 else {
1952 seq_puts(m, ",");
30350d65
TZ		 1953 hist_field_print(m, field);
1954 }
1955 }
1956
1957 if (have_var) {
1958 unsigned int n = 0;
1959
1960 seq_puts(m, ":");
1961
1962 for_each_hist_val_field(i, hist_data) {
1963 field = hist_data->fields[i];
1964
1965 if (field->flags & HIST_FIELD_FL_VAR) {
1966 if (n++)
1967 seq_puts(m, ",");
1968 hist_field_print(m, field);
1969 }
f2606835
TZ		 1970 }
1971 }
7ef224d1
TZ		 1972
1973 seq_puts(m, ":sort=");
e62347d2
TZ		 1974
1975 for (i = 0; i < hist_data->n_sort_keys; i++) {
1976 struct tracing_map_sort_key *sort_key;
30350d65
TZ		 1977 unsigned int idx, first_key_idx;
1978
1979 /* skip VAR vals */
1980 first_key_idx = hist_data->n_vals - hist_data->n_vars;
e62347d2
TZ		 1981
1982 sort_key = &hist_data->sort_keys[i];
ad42febe
TZ		 1983 idx = sort_key->field_idx;
1984
1a361dfc 1985 if (WARN_ON(idx >= HIST_FIELDS_MAX))
ad42febe 1986 return -EINVAL;
e62347d2
TZ		 1987
1988 if (i > 0)
1989 seq_puts(m, ",");
1990
ad42febe 1991 if (idx == HITCOUNT_IDX)
e62347d2 1992 seq_puts(m, "hitcount");
30350d65
TZ		 1993 else {
1994 if (idx >= first_key_idx)
1995 idx += hist_data->n_vars;
e62347d2 1996 hist_field_print(m, hist_data->fields[idx]);
30350d65 1997 }
e62347d2
TZ		 1998
1999 if (sort_key->descending)
2000 seq_puts(m, ".descending");
2001 }
7ef224d1
TZ		 2002 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
2003
2004 if (data->filter_str)
2005 seq_printf(m, " if %s", data->filter_str);
2006
83e99914
TZ		 2007 if (data->paused)
2008 seq_puts(m, " [paused]");
2009 else
2010 seq_puts(m, " [active]");
7ef224d1
TZ		 2011
2012 seq_putc(m, '\n');
2013
2014 return 0;
2015}
2016
5463bfda
TZ		 2017static int event_hist_trigger_init(struct event_trigger_ops *ops,
2018 struct event_trigger_data *data)
2019{
2020 struct hist_trigger_data *hist_data = data->private_data;
2021
2022 if (!data->ref && hist_data->attrs->name)
2023 save_named_trigger(hist_data->attrs->name, data);
2024
2025 data->ref++;
2026
2027 return 0;
2028}
2029
7ef224d1
TZ		 2030static void event_hist_trigger_free(struct event_trigger_ops *ops,
2031 struct event_trigger_data *data)
2032{
2033 struct hist_trigger_data *hist_data = data->private_data;
2034
2035 if (WARN_ON_ONCE(data->ref <= 0))
2036 return;
2037
2038 data->ref--;
2039 if (!data->ref) {
5463bfda
TZ		 2040 if (data->name)
2041 del_named_trigger(data);
7ef224d1
TZ		 2042 trigger_data_free(data);
2043 destroy_hist_data(hist_data);
2044 }
2045}
2046
2047static struct event_trigger_ops event_hist_trigger_ops = {
2048 .func = event_hist_trigger,
2049 .print = event_hist_trigger_print,
5463bfda 2050 .init = event_hist_trigger_init,
7ef224d1
TZ		 2051 .free = event_hist_trigger_free,
2052};
2053
5463bfda
TZ		 2054static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
2055 struct event_trigger_data *data)
2056{
2057 data->ref++;
2058
2059 save_named_trigger(data->named_data->name, data);
2060
2061 event_hist_trigger_init(ops, data->named_data);
2062
2063 return 0;
2064}
2065
2066static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
2067 struct event_trigger_data *data)
2068{
2069 if (WARN_ON_ONCE(data->ref <= 0))
2070 return;
2071
2072 event_hist_trigger_free(ops, data->named_data);
2073
2074 data->ref--;
2075 if (!data->ref) {
2076 del_named_trigger(data);
2077 trigger_data_free(data);
2078 }
2079}
2080
2081static struct event_trigger_ops event_hist_trigger_named_ops = {
2082 .func = event_hist_trigger,
2083 .print = event_hist_trigger_print,
2084 .init = event_hist_trigger_named_init,
2085 .free = event_hist_trigger_named_free,
2086};
2087
7ef224d1
TZ		 2088static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
2089 char *param)
2090{
2091 return &event_hist_trigger_ops;
2092}
2093
e86ae9ba
TZ		 2094static void hist_clear(struct event_trigger_data *data)
2095{
2096 struct hist_trigger_data *hist_data = data->private_data;
e86ae9ba 2097
5463bfda
TZ		 2098 if (data->name)
2099 pause_named_trigger(data);
e86ae9ba
TZ		 2100
2101 synchronize_sched();
2102
2103 tracing_map_clear(hist_data->map);
2104
5463bfda
TZ		 2105 if (data->name)
2106 unpause_named_trigger(data);
2107}
2108
2109static bool compatible_field(struct ftrace_event_field *field,
2110 struct ftrace_event_field *test_field)
2111{
2112 if (field == test_field)
2113 return true;
2114 if (field == NULL || test_field == NULL)
2115 return false;
2116 if (strcmp(field->name, test_field->name) != 0)
2117 return false;
2118 if (strcmp(field->type, test_field->type) != 0)
2119 return false;
2120 if (field->size != test_field->size)
2121 return false;
2122 if (field->is_signed != test_field->is_signed)
2123 return false;
2124
2125 return true;
e86ae9ba
TZ		 2126}
2127
52a7f16d 2128static bool hist_trigger_match(struct event_trigger_data *data,
5463bfda
TZ		 2129 struct event_trigger_data *data_test,
2130 struct event_trigger_data *named_data,
2131 bool ignore_filter)
52a7f16d
TZ		 2132{
2133 struct tracing_map_sort_key *sort_key, *sort_key_test;
2134 struct hist_trigger_data *hist_data, *hist_data_test;
2135 struct hist_field *key_field, *key_field_test;
2136 unsigned int i;
2137
5463bfda
TZ		 2138 if (named_data && (named_data != data_test) &&
2139 (named_data != data_test->named_data))
2140 return false;
2141
2142 if (!named_data && is_named_trigger(data_test))
2143 return false;
2144
52a7f16d
TZ		 2145 hist_data = data->private_data;
2146 hist_data_test = data_test->private_data;
2147
2148 if (hist_data->n_vals != hist_data_test->n_vals ||
2149 hist_data->n_fields != hist_data_test->n_fields ||
2150 hist_data->n_sort_keys != hist_data_test->n_sort_keys)
2151 return false;
2152
5463bfda
TZ		 2153 if (!ignore_filter) {
2154 if ((data->filter_str && !data_test->filter_str) ||
2155 (!data->filter_str && data_test->filter_str))
2156 return false;
2157 }
52a7f16d
TZ		 2158
2159 for_each_hist_field(i, hist_data) {
2160 key_field = hist_data->fields[i];
2161 key_field_test = hist_data_test->fields[i];
2162
2163 if (key_field->flags != key_field_test->flags)
2164 return false;
5463bfda 2165 if (!compatible_field(key_field->field, key_field_test->field))
52a7f16d
TZ		 2166 return false;
2167 if (key_field->offset != key_field_test->offset)
2168 return false;
ad42febe
TZ		 2169 if (key_field->size != key_field_test->size)
2170 return false;
2171 if (key_field->is_signed != key_field_test->is_signed)
2172 return false;
1a361dfc
TZ		 2173 if (!!key_field->var.name != !!key_field_test->var.name)
2174 return false;
2175 if (key_field->var.name &&
2176 strcmp(key_field->var.name, key_field_test->var.name) != 0)
2177 return false;
52a7f16d
TZ		 2178 }
2179
2180 for (i = 0; i < hist_data->n_sort_keys; i++) {
2181 sort_key = &hist_data->sort_keys[i];
2182 sort_key_test = &hist_data_test->sort_keys[i];
2183
2184 if (sort_key->field_idx != sort_key_test->field_idx ||
2185 sort_key->descending != sort_key_test->descending)
2186 return false;
2187 }
2188
5463bfda 2189 if (!ignore_filter && data->filter_str &&
52a7f16d
TZ		 2190 (strcmp(data->filter_str, data_test->filter_str) != 0))
2191 return false;
2192
2193 return true;
2194}
2195
7ef224d1
TZ		 2196static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
2197 struct event_trigger_data *data,
2198 struct trace_event_file *file)
2199{
83e99914 2200 struct hist_trigger_data *hist_data = data->private_data;
5463bfda 2201 struct event_trigger_data *test, *named_data = NULL;
7ef224d1
TZ		 2202 int ret = 0;
2203
5463bfda
TZ		 2204 if (hist_data->attrs->name) {
2205 named_data = find_named_trigger(hist_data->attrs->name);
2206 if (named_data) {
2207 if (!hist_trigger_match(data, named_data, named_data,
2208 true)) {
2209 ret = -EINVAL;
2210 goto out;
2211 }
2212 }
2213 }
2214
2215 if (hist_data->attrs->name && !named_data)
2216 goto new;
2217
7ef224d1
TZ		 2218 list_for_each_entry_rcu(test, &file->triggers, list) {
2219 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 2220 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d 2221 continue;
83e99914
TZ		 2222 if (hist_data->attrs->pause)
2223 test->paused = true;
2224 else if (hist_data->attrs->cont)
2225 test->paused = false;
e86ae9ba
TZ		 2226 else if (hist_data->attrs->clear)
2227 hist_clear(test);
83e99914
TZ		 2228 else
2229 ret = -EEXIST;
7ef224d1
TZ		 2230 goto out;
2231 }
2232 }
5463bfda 2233 new:
e86ae9ba 2234 if (hist_data->attrs->cont || hist_data->attrs->clear) {
83e99914
TZ		 2235 ret = -ENOENT;
2236 goto out;
2237 }
2238
7522c03a
TZ		 2239 if (hist_data->attrs->pause)
2240 data->paused = true;
2241
5463bfda
TZ		 2242 if (named_data) {
2243 destroy_hist_data(data->private_data);
2244 data->private_data = named_data->private_data;
2245 set_named_trigger_data(data, named_data);
2246 data->ops = &event_hist_trigger_named_ops;
2247 }
2248
7ef224d1
TZ		 2249 if (data->ops->init) {
2250 ret = data->ops->init(data->ops, data);
2251 if (ret < 0)
2252 goto out;
2253 }
2254
2255 list_add_rcu(&data->list, &file->triggers);
2256 ret++;
2257
2258 update_cond_flag(file);
5463bfda 2259
ad42febe
TZ		 2260 if (hist_data->enable_timestamps)
2261 tracing_set_time_stamp_abs(file->tr, true);
2262
7ef224d1
TZ		 2263 if (trace_event_trigger_enable_disable(file, 1) < 0) {
2264 list_del_rcu(&data->list);
2265 update_cond_flag(file);
2266 ret--;
2267 }
2268 out:
2269 return ret;
2270}
2271
52a7f16d
TZ		 2272static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
2273 struct event_trigger_data *data,
2274 struct trace_event_file *file)
2275{
5463bfda
TZ		 2276 struct hist_trigger_data *hist_data = data->private_data;
2277 struct event_trigger_data *test, *named_data = NULL;
52a7f16d
TZ		 2278 bool unregistered = false;
2279
5463bfda
TZ		 2280 if (hist_data->attrs->name)
2281 named_data = find_named_trigger(hist_data->attrs->name);
2282
52a7f16d
TZ		 2283 list_for_each_entry_rcu(test, &file->triggers, list) {
2284 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 2285 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d
TZ		 2286 continue;
2287 unregistered = true;
2288 list_del_rcu(&test->list);
2289 trace_event_trigger_enable_disable(file, 0);
2290 update_cond_flag(file);
2291 break;
2292 }
2293 }
2294
2295 if (unregistered && test->ops->free)
2296 test->ops->free(test->ops, test);
ad42febe
TZ		 2297
2298 if (hist_data->enable_timestamps) {
30350d65 2299 if (!hist_data->remove || unregistered)
ad42febe
TZ		 2300 tracing_set_time_stamp_abs(file->tr, false);
2301 }
52a7f16d
TZ		 2302}
2303
2304static void hist_unreg_all(struct trace_event_file *file)
2305{
47c18569 2306 struct event_trigger_data *test, *n;
ad42febe 2307 struct hist_trigger_data *hist_data;
52a7f16d 2308
47c18569 2309 list_for_each_entry_safe(test, n, &file->triggers, list) {
52a7f16d 2310 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
ad42febe 2311 hist_data = test->private_data;
52a7f16d
TZ		 2312 list_del_rcu(&test->list);
2313 trace_event_trigger_enable_disable(file, 0);
2314 update_cond_flag(file);
ad42febe
TZ		 2315 if (hist_data->enable_timestamps)
2316 tracing_set_time_stamp_abs(file->tr, false);
52a7f16d
TZ		 2317 if (test->ops->free)
2318 test->ops->free(test->ops, test);
2319 }
2320 }
2321}
2322
7ef224d1
TZ		 2323static int event_hist_trigger_func(struct event_command *cmd_ops,
2324 struct trace_event_file *file,
2325 char *glob, char *cmd, char *param)
2326{
2327 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
2328 struct event_trigger_data *trigger_data;
2329 struct hist_trigger_attrs *attrs;
2330 struct event_trigger_ops *trigger_ops;
2331 struct hist_trigger_data *hist_data;
30350d65 2332 bool remove = false;
7ef224d1
TZ		 2333 char *trigger;
2334 int ret = 0;
2335
2336 if (!param)
2337 return -EINVAL;
2338
30350d65
TZ		 2339 if (glob[0] == '!')
2340 remove = true;
2341
7ef224d1
TZ		 2342 /* separate the trigger from the filter (k:v [if filter]) */
2343 trigger = strsep(&param, " \t");
2344 if (!trigger)
2345 return -EINVAL;
2346
2347 attrs = parse_hist_trigger_attrs(trigger);
2348 if (IS_ERR(attrs))
2349 return PTR_ERR(attrs);
2350
2351 if (attrs->map_bits)
2352 hist_trigger_bits = attrs->map_bits;
2353
30350d65 2354 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
7ef224d1
TZ		 2355 if (IS_ERR(hist_data)) {
2356 destroy_hist_trigger_attrs(attrs);
2357 return PTR_ERR(hist_data);
2358 }
2359
2360 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
2361
2362 ret = -ENOMEM;
2363 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
2364 if (!trigger_data)
2365 goto out_free;
2366
2367 trigger_data->count = -1;
2368 trigger_data->ops = trigger_ops;
2369 trigger_data->cmd_ops = cmd_ops;
2370
2371 INIT_LIST_HEAD(&trigger_data->list);
2372 RCU_INIT_POINTER(trigger_data->filter, NULL);
2373
2374 trigger_data->private_data = hist_data;
2375
52a7f16d
TZ		 2376 /* if param is non-empty, it's supposed to be a filter */
2377 if (param && cmd_ops->set_filter) {
2378 ret = cmd_ops->set_filter(param, trigger_data, file);
2379 if (ret < 0)
2380 goto out_free;
2381 }
2382
30350d65 2383 if (remove) {
7ef224d1
TZ		 2384 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
2385 ret = 0;
2386 goto out_free;
2387 }
2388
7ef224d1
TZ		 2389 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
2390 /*
2391 * The above returns on success the # of triggers registered,
2392 * but if it didn't register any it returns zero. Consider no
2393 * triggers registered a failure too.
2394 */
2395 if (!ret) {
e86ae9ba 2396 if (!(attrs->pause || attrs->cont || attrs->clear))
83e99914 2397 ret = -ENOENT;
7ef224d1
TZ		 2398 goto out_free;
2399 } else if (ret < 0)
2400 goto out_free;
2401 /* Just return zero, not the number of registered triggers */
2402 ret = 0;
2403 out:
2404 return ret;
2405 out_free:
2406 if (cmd_ops->set_filter)
2407 cmd_ops->set_filter(NULL, trigger_data, NULL);
2408
2409 kfree(trigger_data);
2410
2411 destroy_hist_data(hist_data);
2412 goto out;
2413}
2414
2415static struct event_command trigger_hist_cmd = {
2416 .name = "hist",
2417 .trigger_type = ETT_EVENT_HIST,
2418 .flags = EVENT_CMD_FL_NEEDS_REC,
2419 .func = event_hist_trigger_func,
2420 .reg = hist_register_trigger,
52a7f16d
TZ		 2421 .unreg = hist_unregister_trigger,
2422 .unreg_all = hist_unreg_all,
7ef224d1
TZ		 2423 .get_trigger_ops = event_hist_get_trigger_ops,
2424 .set_filter = set_trigger_filter,
2425};
2426
2427__init int register_trigger_hist_cmd(void)
2428{
2429 int ret;
2430
2431 ret = register_event_command(&trigger_hist_cmd);
2432 WARN_ON(ret < 0);
2433
2434 return ret;
2435}
d0bad49b
TZ		 2436
2437static void
1ac4f51c
TZ		 2438hist_enable_trigger(struct event_trigger_data *data, void *rec,
2439 struct ring_buffer_event *event)
d0bad49b
TZ		 2440{
2441 struct enable_trigger_data *enable_data = data->private_data;
2442 struct event_trigger_data *test;
2443
2444 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
2445 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2446 if (enable_data->enable)
2447 test->paused = false;
2448 else
2449 test->paused = true;
d0bad49b
TZ		 2450 }
2451 }
2452}
2453
2454static void
1ac4f51c
TZ		 2455hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
2456 struct ring_buffer_event *event)
d0bad49b
TZ		 2457{
2458 if (!data->count)
2459 return;
2460
2461 if (data->count != -1)
2462 (data->count)--;
2463
1ac4f51c 2464 hist_enable_trigger(data, rec, event);
d0bad49b
TZ		 2465}
2466
2467static struct event_trigger_ops hist_enable_trigger_ops = {
2468 .func = hist_enable_trigger,
2469 .print = event_enable_trigger_print,
2470 .init = event_trigger_init,
2471 .free = event_enable_trigger_free,
2472};
2473
2474static struct event_trigger_ops hist_enable_count_trigger_ops = {
2475 .func = hist_enable_count_trigger,
2476 .print = event_enable_trigger_print,
2477 .init = event_trigger_init,
2478 .free = event_enable_trigger_free,
2479};
2480
2481static struct event_trigger_ops hist_disable_trigger_ops = {
2482 .func = hist_enable_trigger,
2483 .print = event_enable_trigger_print,
2484 .init = event_trigger_init,
2485 .free = event_enable_trigger_free,
2486};
2487
2488static struct event_trigger_ops hist_disable_count_trigger_ops = {
2489 .func = hist_enable_count_trigger,
2490 .print = event_enable_trigger_print,
2491 .init = event_trigger_init,
2492 .free = event_enable_trigger_free,
2493};
2494
2495static struct event_trigger_ops *
2496hist_enable_get_trigger_ops(char *cmd, char *param)
2497{
2498 struct event_trigger_ops *ops;
2499 bool enable;
2500
2501 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
2502
2503 if (enable)
2504 ops = param ? &hist_enable_count_trigger_ops :
2505 &hist_enable_trigger_ops;
2506 else
2507 ops = param ? &hist_disable_count_trigger_ops :
2508 &hist_disable_trigger_ops;
2509
2510 return ops;
2511}
2512
52a7f16d
TZ		 2513static void hist_enable_unreg_all(struct trace_event_file *file)
2514{
47c18569 2515 struct event_trigger_data *test, *n;
52a7f16d 2516
47c18569 2517 list_for_each_entry_safe(test, n, &file->triggers, list) {
52a7f16d
TZ		 2518 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
2519 list_del_rcu(&test->list);
2520 update_cond_flag(file);
2521 trace_event_trigger_enable_disable(file, 0);
2522 if (test->ops->free)
2523 test->ops->free(test->ops, test);
2524 }
2525 }
2526}
2527
d0bad49b
TZ		 2528static struct event_command trigger_hist_enable_cmd = {
2529 .name = ENABLE_HIST_STR,
2530 .trigger_type = ETT_HIST_ENABLE,
2531 .func = event_enable_trigger_func,
2532 .reg = event_enable_register_trigger,
2533 .unreg = event_enable_unregister_trigger,
52a7f16d 2534 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 2535 .get_trigger_ops = hist_enable_get_trigger_ops,
2536 .set_filter = set_trigger_filter,
2537};
2538
2539static struct event_command trigger_hist_disable_cmd = {
2540 .name = DISABLE_HIST_STR,
2541 .trigger_type = ETT_HIST_ENABLE,
2542 .func = event_enable_trigger_func,
2543 .reg = event_enable_register_trigger,
2544 .unreg = event_enable_unregister_trigger,
52a7f16d 2545 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 2546 .get_trigger_ops = hist_enable_get_trigger_ops,
2547 .set_filter = set_trigger_filter,
2548};
2549
2550static __init void unregister_trigger_hist_enable_disable_cmds(void)
2551{
2552 unregister_event_command(&trigger_hist_enable_cmd);
2553 unregister_event_command(&trigger_hist_disable_cmd);
2554}
2555
2556__init int register_trigger_hist_enable_disable_cmds(void)
2557{
2558 int ret;
2559
2560 ret = register_event_command(&trigger_hist_enable_cmd);
2561 if (WARN_ON(ret < 0))
2562 return ret;
2563 ret = register_event_command(&trigger_hist_disable_cmd);
2564 if (WARN_ON(ret < 0))
2565 unregister_trigger_hist_enable_disable_cmds();
2566
2567 return ret;
2568}
Linux 6.x block layer and io_uring tree(s)