projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tracing: Add variable reference handling to hist triggers
[linux-2.6-block.git] / kernel / trace / trace_events_hist.c
CommitLineData
7ef224d1
TZ		 1/*
2 * trace_events_hist - trace event hist triggers
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
15 */
16
17#include <linux/module.h>
18#include <linux/kallsyms.h>
19#include <linux/mutex.h>
20#include <linux/slab.h>
21#include <linux/stacktrace.h>
b2d09103 22#include <linux/rculist.h>
7ef224d1
TZ		 23
24#include "tracing_map.h"
25#include "trace.h"
26
27struct hist_field;
28
df35d93b
TZ		 29typedef u64 (*hist_field_fn_t) (struct hist_field *field,
30 struct tracing_map_elt *elt,
31 struct ring_buffer_event *rbe,
32 void *event);
7ef224d1 33
5819eadd 34#define HIST_FIELD_OPERANDS_MAX 2
30350d65
TZ		 35#define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
36
100719dc
TZ		 37enum field_op_id {
38 FIELD_OP_NONE,
39 FIELD_OP_PLUS,
40 FIELD_OP_MINUS,
41 FIELD_OP_UNARY_MINUS,
42};
43
30350d65
TZ		 44struct hist_var {
45 char *name;
46 struct hist_trigger_data *hist_data;
47 unsigned int idx;
48};
5819eadd 49
7ef224d1
TZ		 50struct hist_field {
51 struct ftrace_event_field *field;
52 unsigned long flags;
53 hist_field_fn_t fn;
54 unsigned int size;
76a3b0c8 55 unsigned int offset;
5819eadd 56 unsigned int is_signed;
19a9facd 57 const char *type;
5819eadd 58 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
b559d003 59 struct hist_trigger_data *hist_data;
30350d65 60 struct hist_var var;
100719dc 61 enum field_op_id operator;
067fe038
TZ		 62 char *system;
63 char *event_name;
100719dc 64 char *name;
067fe038
TZ		 65 unsigned int var_idx;
66 unsigned int var_ref_idx;
67 bool read_once;
7ef224d1
TZ		 68};
69
df35d93b
TZ		 70static u64 hist_field_none(struct hist_field *field,
71 struct tracing_map_elt *elt,
72 struct ring_buffer_event *rbe,
73 void *event)
69a0200c
TZ		 74{
75 return 0;
76}
77
df35d93b
TZ		 78static u64 hist_field_counter(struct hist_field *field,
79 struct tracing_map_elt *elt,
80 struct ring_buffer_event *rbe,
81 void *event)
7ef224d1
TZ		 82{
83 return 1;
84}
85
df35d93b
TZ		 86static u64 hist_field_string(struct hist_field *hist_field,
87 struct tracing_map_elt *elt,
88 struct ring_buffer_event *rbe,
89 void *event)
7ef224d1
TZ		 90{
91 char *addr = (char *)(event + hist_field->field->offset);
92
93 return (u64)(unsigned long)addr;
94}
95
df35d93b
TZ		 96static u64 hist_field_dynstring(struct hist_field *hist_field,
97 struct tracing_map_elt *elt,
98 struct ring_buffer_event *rbe,
99 void *event)
79e577cb
NK		 100{
101 u32 str_item = *(u32 *)(event + hist_field->field->offset);
102 int str_loc = str_item & 0xffff;
103 char *addr = (char *)(event + str_loc);
104
105 return (u64)(unsigned long)addr;
106}
107
df35d93b
TZ		 108static u64 hist_field_pstring(struct hist_field *hist_field,
109 struct tracing_map_elt *elt,
110 struct ring_buffer_event *rbe,
111 void *event)
79e577cb
NK		 112{
113 char **addr = (char **)(event + hist_field->field->offset);
114
115 return (u64)(unsigned long)*addr;
116}
117
df35d93b
TZ		 118static u64 hist_field_log2(struct hist_field *hist_field,
119 struct tracing_map_elt *elt,
120 struct ring_buffer_event *rbe,
121 void *event)
4b94f5b7 122{
5819eadd
TZ		 123 struct hist_field *operand = hist_field->operands[0];
124
df35d93b 125 u64 val = operand->fn(operand, elt, rbe, event);
4b94f5b7
NK		 126
127 return (u64) ilog2(roundup_pow_of_two(val));
128}
129
df35d93b
TZ		 130static u64 hist_field_plus(struct hist_field *hist_field,
131 struct tracing_map_elt *elt,
132 struct ring_buffer_event *rbe,
133 void *event)
100719dc
TZ		 134{
135 struct hist_field *operand1 = hist_field->operands[0];
136 struct hist_field *operand2 = hist_field->operands[1];
137
df35d93b
TZ		 138 u64 val1 = operand1->fn(operand1, elt, rbe, event);
139 u64 val2 = operand2->fn(operand2, elt, rbe, event);
100719dc
TZ		 140
141 return val1 + val2;
142}
143
df35d93b
TZ		 144static u64 hist_field_minus(struct hist_field *hist_field,
145 struct tracing_map_elt *elt,
146 struct ring_buffer_event *rbe,
147 void *event)
100719dc
TZ		 148{
149 struct hist_field *operand1 = hist_field->operands[0];
150 struct hist_field *operand2 = hist_field->operands[1];
151
df35d93b
TZ		 152 u64 val1 = operand1->fn(operand1, elt, rbe, event);
153 u64 val2 = operand2->fn(operand2, elt, rbe, event);
100719dc
TZ		 154
155 return val1 - val2;
156}
157
df35d93b
TZ		 158static u64 hist_field_unary_minus(struct hist_field *hist_field,
159 struct tracing_map_elt *elt,
160 struct ring_buffer_event *rbe,
161 void *event)
100719dc
TZ		 162{
163 struct hist_field *operand = hist_field->operands[0];
164
df35d93b 165 s64 sval = (s64)operand->fn(operand, elt, rbe, event);
100719dc
TZ		 166 u64 val = (u64)-sval;
167
168 return val;
169}
170
7ef224d1 171#define DEFINE_HIST_FIELD_FN(type) \
fbd302cb 172 static u64 hist_field_##type(struct hist_field *hist_field, \
df35d93b
TZ		 173 struct tracing_map_elt *elt, \
174 struct ring_buffer_event *rbe, \
175 void *event) \
7ef224d1
TZ		 176{ \
177 type *addr = (type *)(event + hist_field->field->offset); \
178 \
79e577cb 179 return (u64)(unsigned long)*addr; \
7ef224d1
TZ		 180}
181
182DEFINE_HIST_FIELD_FN(s64);
183DEFINE_HIST_FIELD_FN(u64);
184DEFINE_HIST_FIELD_FN(s32);
185DEFINE_HIST_FIELD_FN(u32);
186DEFINE_HIST_FIELD_FN(s16);
187DEFINE_HIST_FIELD_FN(u16);
188DEFINE_HIST_FIELD_FN(s8);
189DEFINE_HIST_FIELD_FN(u8);
190
191#define for_each_hist_field(i, hist_data) \
192 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
193
194#define for_each_hist_val_field(i, hist_data) \
195 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
196
197#define for_each_hist_key_field(i, hist_data) \
198 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
199
69a0200c
TZ		 200#define HIST_STACKTRACE_DEPTH 16
201#define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
202#define HIST_STACKTRACE_SKIP 5
203
7ef224d1 204#define HITCOUNT_IDX 0
69a0200c 205#define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
7ef224d1
TZ		 206
207enum hist_field_flags {
0d7a8325
TZ		 208 HIST_FIELD_FL_HITCOUNT = 1 << 0,
209 HIST_FIELD_FL_KEY = 1 << 1,
210 HIST_FIELD_FL_STRING = 1 << 2,
211 HIST_FIELD_FL_HEX = 1 << 3,
212 HIST_FIELD_FL_SYM = 1 << 4,
213 HIST_FIELD_FL_SYM_OFFSET = 1 << 5,
214 HIST_FIELD_FL_EXECNAME = 1 << 6,
215 HIST_FIELD_FL_SYSCALL = 1 << 7,
216 HIST_FIELD_FL_STACKTRACE = 1 << 8,
217 HIST_FIELD_FL_LOG2 = 1 << 9,
ad42febe 218 HIST_FIELD_FL_TIMESTAMP = 1 << 10,
860f9f6b 219 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11,
30350d65 220 HIST_FIELD_FL_VAR = 1 << 12,
100719dc 221 HIST_FIELD_FL_EXPR = 1 << 13,
067fe038 222 HIST_FIELD_FL_VAR_REF = 1 << 14,
30350d65
TZ		 223};
224
225struct var_defs {
226 unsigned int n_vars;
227 char *name[TRACING_MAP_VARS_MAX];
228 char *expr[TRACING_MAP_VARS_MAX];
7ef224d1
TZ		 229};
230
231struct hist_trigger_attrs {
232 char *keys_str;
f2606835 233 char *vals_str;
e62347d2 234 char *sort_key_str;
5463bfda 235 char *name;
83e99914
TZ		 236 bool pause;
237 bool cont;
e86ae9ba 238 bool clear;
860f9f6b 239 bool ts_in_usecs;
7ef224d1 240 unsigned int map_bits;
30350d65
TZ		 241
242 char *assignment_str[TRACING_MAP_VARS_MAX];
243 unsigned int n_assignments;
244
245 struct var_defs var_defs;
7ef224d1
TZ		 246};
247
248struct hist_trigger_data {
30350d65 249 struct hist_field *fields[HIST_FIELDS_MAX];
7ef224d1
TZ		 250 unsigned int n_vals;
251 unsigned int n_keys;
252 unsigned int n_fields;
30350d65 253 unsigned int n_vars;
7ef224d1
TZ		 254 unsigned int key_size;
255 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
256 unsigned int n_sort_keys;
257 struct trace_event_file *event_file;
258 struct hist_trigger_attrs *attrs;
259 struct tracing_map *map;
ad42febe 260 bool enable_timestamps;
30350d65 261 bool remove;
067fe038
TZ		 262 struct hist_field *var_refs[TRACING_MAP_VARS_MAX];
263 unsigned int n_var_refs;
7ef224d1
TZ		 264};
265
df35d93b
TZ		 266static u64 hist_field_timestamp(struct hist_field *hist_field,
267 struct tracing_map_elt *elt,
268 struct ring_buffer_event *rbe,
269 void *event)
860f9f6b
TZ		 270{
271 struct hist_trigger_data *hist_data = hist_field->hist_data;
272 struct trace_array *tr = hist_data->event_file->tr;
273
274 u64 ts = ring_buffer_event_time_stamp(rbe);
275
276 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
277 ts = ns2usecs(ts);
278
279 return ts;
280}
281
067fe038
TZ		 282struct hist_var_data {
283 struct list_head list;
284 struct hist_trigger_data *hist_data;
285};
286
287static struct hist_field *
288check_field_for_var_ref(struct hist_field *hist_field,
289 struct hist_trigger_data *var_data,
290 unsigned int var_idx)
291{
292 struct hist_field *found = NULL;
293
294 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) {
295 if (hist_field->var.idx == var_idx &&
296 hist_field->var.hist_data == var_data) {
297 found = hist_field;
298 }
299 }
300
301 return found;
302}
303
304static struct hist_field *
305check_field_for_var_refs(struct hist_trigger_data *hist_data,
306 struct hist_field *hist_field,
307 struct hist_trigger_data *var_data,
308 unsigned int var_idx,
309 unsigned int level)
310{
311 struct hist_field *found = NULL;
312 unsigned int i;
313
314 if (level > 3)
315 return found;
316
317 if (!hist_field)
318 return found;
319
320 found = check_field_for_var_ref(hist_field, var_data, var_idx);
321 if (found)
322 return found;
323
324 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
325 struct hist_field *operand;
326
327 operand = hist_field->operands[i];
328 found = check_field_for_var_refs(hist_data, operand, var_data,
329 var_idx, level + 1);
330 if (found)
331 return found;
332 }
333
334 return found;
335}
336
337static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
338 struct hist_trigger_data *var_data,
339 unsigned int var_idx)
340{
341 struct hist_field *hist_field, *found = NULL;
342 unsigned int i;
343
344 for_each_hist_field(i, hist_data) {
345 hist_field = hist_data->fields[i];
346 found = check_field_for_var_refs(hist_data, hist_field,
347 var_data, var_idx, 0);
348 if (found)
349 return found;
350 }
351
352 return found;
353}
354
355static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
356 unsigned int var_idx)
357{
358 struct trace_array *tr = hist_data->event_file->tr;
359 struct hist_field *found = NULL;
360 struct hist_var_data *var_data;
361
362 list_for_each_entry(var_data, &tr->hist_vars, list) {
363 if (var_data->hist_data == hist_data)
364 continue;
365 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
366 if (found)
367 break;
368 }
369
370 return found;
371}
372
373static bool check_var_refs(struct hist_trigger_data *hist_data)
374{
375 struct hist_field *field;
376 bool found = false;
377 int i;
378
379 for_each_hist_field(i, hist_data) {
380 field = hist_data->fields[i];
381 if (field && field->flags & HIST_FIELD_FL_VAR) {
382 if (find_any_var_ref(hist_data, field->var.idx)) {
383 found = true;
384 break;
385 }
386 }
387 }
388
389 return found;
390}
391
392static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
393{
394 struct trace_array *tr = hist_data->event_file->tr;
395 struct hist_var_data *var_data, *found = NULL;
396
397 list_for_each_entry(var_data, &tr->hist_vars, list) {
398 if (var_data->hist_data == hist_data) {
399 found = var_data;
400 break;
401 }
402 }
403
404 return found;
405}
406
407static bool field_has_hist_vars(struct hist_field *hist_field,
408 unsigned int level)
409{
410 int i;
411
412 if (level > 3)
413 return false;
414
415 if (!hist_field)
416 return false;
417
418 if (hist_field->flags & HIST_FIELD_FL_VAR ||
419 hist_field->flags & HIST_FIELD_FL_VAR_REF)
420 return true;
421
422 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
423 struct hist_field *operand;
424
425 operand = hist_field->operands[i];
426 if (field_has_hist_vars(operand, level + 1))
427 return true;
428 }
429
430 return false;
431}
432
433static bool has_hist_vars(struct hist_trigger_data *hist_data)
434{
435 struct hist_field *hist_field;
436 int i;
437
438 for_each_hist_field(i, hist_data) {
439 hist_field = hist_data->fields[i];
440 if (field_has_hist_vars(hist_field, 0))
441 return true;
442 }
443
444 return false;
445}
446
447static int save_hist_vars(struct hist_trigger_data *hist_data)
448{
449 struct trace_array *tr = hist_data->event_file->tr;
450 struct hist_var_data *var_data;
451
452 var_data = find_hist_vars(hist_data);
453 if (var_data)
454 return 0;
455
456 if (trace_array_get(tr) < 0)
457 return -ENODEV;
458
459 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
460 if (!var_data) {
461 trace_array_put(tr);
462 return -ENOMEM;
463 }
464
465 var_data->hist_data = hist_data;
466 list_add(&var_data->list, &tr->hist_vars);
467
468 return 0;
469}
470
471static void remove_hist_vars(struct hist_trigger_data *hist_data)
472{
473 struct trace_array *tr = hist_data->event_file->tr;
474 struct hist_var_data *var_data;
475
476 var_data = find_hist_vars(hist_data);
477 if (!var_data)
478 return;
479
480 if (WARN_ON(check_var_refs(hist_data)))
481 return;
482
483 list_del(&var_data->list);
484
485 kfree(var_data);
486
487 trace_array_put(tr);
488}
489
30350d65
TZ		 490static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
491 const char *var_name)
492{
493 struct hist_field *hist_field, *found = NULL;
494 int i;
495
496 for_each_hist_field(i, hist_data) {
497 hist_field = hist_data->fields[i];
498 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
499 strcmp(hist_field->var.name, var_name) == 0) {
500 found = hist_field;
501 break;
502 }
503 }
504
505 return found;
506}
507
508static struct hist_field *find_var(struct hist_trigger_data *hist_data,
509 struct trace_event_file *file,
510 const char *var_name)
511{
512 struct hist_trigger_data *test_data;
513 struct event_trigger_data *test;
514 struct hist_field *hist_field;
515
516 hist_field = find_var_field(hist_data, var_name);
517 if (hist_field)
518 return hist_field;
519
520 list_for_each_entry_rcu(test, &file->triggers, list) {
521 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
522 test_data = test->private_data;
523 hist_field = find_var_field(test_data, var_name);
524 if (hist_field)
525 return hist_field;
526 }
527 }
528
529 return NULL;
530}
531
067fe038
TZ		 532static struct trace_event_file *find_var_file(struct trace_array *tr,
533 char *system,
534 char *event_name,
535 char *var_name)
536{
537 struct hist_trigger_data *var_hist_data;
538 struct hist_var_data *var_data;
539 struct trace_event_file *file, *found = NULL;
540
541 if (system)
542 return find_event_file(tr, system, event_name);
543
544 list_for_each_entry(var_data, &tr->hist_vars, list) {
545 var_hist_data = var_data->hist_data;
546 file = var_hist_data->event_file;
547 if (file == found)
548 continue;
549
550 if (find_var_field(var_hist_data, var_name)) {
551 if (found)
552 return NULL;
553
554 found = file;
555 }
556 }
557
558 return found;
559}
560
561static struct hist_field *find_file_var(struct trace_event_file *file,
562 const char *var_name)
563{
564 struct hist_trigger_data *test_data;
565 struct event_trigger_data *test;
566 struct hist_field *hist_field;
567
568 list_for_each_entry_rcu(test, &file->triggers, list) {
569 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
570 test_data = test->private_data;
571 hist_field = find_var_field(test_data, var_name);
572 if (hist_field)
573 return hist_field;
574 }
575 }
576
577 return NULL;
578}
579
580static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
581 char *system,
582 char *event_name,
583 char *var_name)
584{
585 struct trace_array *tr = hist_data->event_file->tr;
586 struct hist_field *hist_field = NULL;
587 struct trace_event_file *file;
588
589 file = find_var_file(tr, system, event_name, var_name);
590 if (!file)
591 return NULL;
592
593 hist_field = find_file_var(file, var_name);
594
595 return hist_field;
596}
597
af6a29bc
TZ		 598struct hist_elt_data {
599 char *comm;
067fe038 600 u64 *var_ref_vals;
af6a29bc
TZ		 601};
602
067fe038
TZ		 603static u64 hist_field_var_ref(struct hist_field *hist_field,
604 struct tracing_map_elt *elt,
605 struct ring_buffer_event *rbe,
606 void *event)
607{
608 struct hist_elt_data *elt_data;
609 u64 var_val = 0;
610
611 elt_data = elt->private_data;
612 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
613
614 return var_val;
615}
616
617static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
618 u64 *var_ref_vals, bool self)
619{
620 struct hist_trigger_data *var_data;
621 struct tracing_map_elt *var_elt;
622 struct hist_field *hist_field;
623 unsigned int i, var_idx;
624 bool resolved = true;
625 u64 var_val = 0;
626
627 for (i = 0; i < hist_data->n_var_refs; i++) {
628 hist_field = hist_data->var_refs[i];
629 var_idx = hist_field->var.idx;
630 var_data = hist_field->var.hist_data;
631
632 if (var_data == NULL) {
633 resolved = false;
634 break;
635 }
636
637 if ((self && var_data != hist_data) ||
638 (!self && var_data == hist_data))
639 continue;
640
641 var_elt = tracing_map_lookup(var_data->map, key);
642 if (!var_elt) {
643 resolved = false;
644 break;
645 }
646
647 if (!tracing_map_var_set(var_elt, var_idx)) {
648 resolved = false;
649 break;
650 }
651
652 if (self || !hist_field->read_once)
653 var_val = tracing_map_read_var(var_elt, var_idx);
654 else
655 var_val = tracing_map_read_var_once(var_elt, var_idx);
656
657 var_ref_vals[i] = var_val;
658 }
659
660 return resolved;
661}
662
85013256
TZ		 663static const char *hist_field_name(struct hist_field *field,
664 unsigned int level)
665{
666 const char *field_name = "";
667
668 if (level > 1)
669 return field_name;
670
671 if (field->field)
672 field_name = field->field->name;
5819eadd
TZ		 673 else if (field->flags & HIST_FIELD_FL_LOG2)
674 field_name = hist_field_name(field->operands[0], ++level);
ad42febe
TZ		 675 else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
676 field_name = "common_timestamp";
067fe038
TZ		 677 else if (field->flags & HIST_FIELD_FL_EXPR ||
678 field->flags & HIST_FIELD_FL_VAR_REF) {
679 if (field->system) {
680 static char full_name[MAX_FILTER_STR_VAL];
681
682 strcat(full_name, field->system);
683 strcat(full_name, ".");
684 strcat(full_name, field->event_name);
685 strcat(full_name, ".");
686 strcat(full_name, field->name);
687 field_name = full_name;
688 } else
689 field_name = field->name;
690 }
85013256
TZ		 691
692 if (field_name == NULL)
693 field_name = "";
694
695 return field_name;
696}
697
7ef224d1
TZ		 698static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
699{
700 hist_field_fn_t fn = NULL;
701
702 switch (field_size) {
703 case 8:
704 if (field_is_signed)
705 fn = hist_field_s64;
706 else
707 fn = hist_field_u64;
708 break;
709 case 4:
710 if (field_is_signed)
711 fn = hist_field_s32;
712 else
713 fn = hist_field_u32;
714 break;
715 case 2:
716 if (field_is_signed)
717 fn = hist_field_s16;
718 else
719 fn = hist_field_u16;
720 break;
721 case 1:
722 if (field_is_signed)
723 fn = hist_field_s8;
724 else
725 fn = hist_field_u8;
726 break;
727 }
728
729 return fn;
730}
731
732static int parse_map_size(char *str)
733{
734 unsigned long size, map_bits;
735 int ret;
736
737 strsep(&str, "=");
738 if (!str) {
739 ret = -EINVAL;
740 goto out;
741 }
742
743 ret = kstrtoul(str, 0, &size);
744 if (ret)
745 goto out;
746
747 map_bits = ilog2(roundup_pow_of_two(size));
748 if (map_bits < TRACING_MAP_BITS_MIN ||
749 map_bits > TRACING_MAP_BITS_MAX)
750 ret = -EINVAL;
751 else
752 ret = map_bits;
753 out:
754 return ret;
755}
756
757static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
758{
30350d65
TZ		 759 unsigned int i;
760
7ef224d1
TZ		 761 if (!attrs)
762 return;
763
30350d65
TZ		 764 for (i = 0; i < attrs->n_assignments; i++)
765 kfree(attrs->assignment_str[i]);
766
5463bfda 767 kfree(attrs->name);
e62347d2 768 kfree(attrs->sort_key_str);
7ef224d1 769 kfree(attrs->keys_str);
f2606835 770 kfree(attrs->vals_str);
7ef224d1
TZ		 771 kfree(attrs);
772}
773
9b1ae035
TZ		 774static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
775{
776 int ret = 0;
777
778 if ((strncmp(str, "key=", strlen("key=")) == 0) ||
779 (strncmp(str, "keys=", strlen("keys=")) == 0)) {
780 attrs->keys_str = kstrdup(str, GFP_KERNEL);
781 if (!attrs->keys_str) {
782 ret = -ENOMEM;
783 goto out;
784 }
785 } else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
786 (strncmp(str, "vals=", strlen("vals=")) == 0) ||
787 (strncmp(str, "values=", strlen("values=")) == 0)) {
788 attrs->vals_str = kstrdup(str, GFP_KERNEL);
789 if (!attrs->vals_str) {
790 ret = -ENOMEM;
791 goto out;
792 }
793 } else if (strncmp(str, "sort=", strlen("sort=")) == 0) {
794 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
795 if (!attrs->sort_key_str) {
796 ret = -ENOMEM;
797 goto out;
798 }
799 } else if (strncmp(str, "name=", strlen("name=")) == 0) {
800 attrs->name = kstrdup(str, GFP_KERNEL);
801 if (!attrs->name) {
802 ret = -ENOMEM;
803 goto out;
804 }
805 } else if (strncmp(str, "size=", strlen("size=")) == 0) {
806 int map_bits = parse_map_size(str);
807
808 if (map_bits < 0) {
809 ret = map_bits;
810 goto out;
811 }
812 attrs->map_bits = map_bits;
30350d65
TZ		 813 } else {
814 char *assignment;
815
816 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
817 ret = -EINVAL;
818 goto out;
819 }
820
821 assignment = kstrdup(str, GFP_KERNEL);
822 if (!assignment) {
823 ret = -ENOMEM;
824 goto out;
825 }
826
827 attrs->assignment_str[attrs->n_assignments++] = assignment;
828 }
9b1ae035
TZ		 829 out:
830 return ret;
831}
832
7ef224d1
TZ		 833static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
834{
835 struct hist_trigger_attrs *attrs;
836 int ret = 0;
837
838 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
839 if (!attrs)
840 return ERR_PTR(-ENOMEM);
841
842 while (trigger_str) {
843 char *str = strsep(&trigger_str, ":");
844
9b1ae035
TZ		 845 if (strchr(str, '=')) {
846 ret = parse_assignment(str, attrs);
847 if (ret)
848 goto free;
849 } else if (strcmp(str, "pause") == 0)
83e99914
TZ		 850 attrs->pause = true;
851 else if ((strcmp(str, "cont") == 0) ||
852 (strcmp(str, "continue") == 0))
853 attrs->cont = true;
e86ae9ba
TZ		 854 else if (strcmp(str, "clear") == 0)
855 attrs->clear = true;
9b1ae035 856 else {
7ef224d1
TZ		 857 ret = -EINVAL;
858 goto free;
859 }
860 }
861
862 if (!attrs->keys_str) {
863 ret = -EINVAL;
864 goto free;
865 }
866
867 return attrs;
868 free:
869 destroy_hist_trigger_attrs(attrs);
870
871 return ERR_PTR(ret);
872}
873
6b4827ad
TZ		 874static inline void save_comm(char *comm, struct task_struct *task)
875{
876 if (!task->pid) {
877 strcpy(comm, "<idle>");
878 return;
879 }
880
881 if (WARN_ON_ONCE(task->pid < 0)) {
882 strcpy(comm, "<XXX>");
883 return;
884 }
885
886 memcpy(comm, task->comm, TASK_COMM_LEN);
887}
888
af6a29bc
TZ		 889static void hist_elt_data_free(struct hist_elt_data *elt_data)
890{
891 kfree(elt_data->comm);
892 kfree(elt_data);
893}
894
895static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
6b4827ad 896{
af6a29bc
TZ		 897 struct hist_elt_data *elt_data = elt->private_data;
898
899 hist_elt_data_free(elt_data);
6b4827ad
TZ		 900}
901
af6a29bc 902static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
6b4827ad
TZ		 903{
904 struct hist_trigger_data *hist_data = elt->map->private_data;
af6a29bc
TZ		 905 unsigned int size = TASK_COMM_LEN;
906 struct hist_elt_data *elt_data;
6b4827ad
TZ		 907 struct hist_field *key_field;
908 unsigned int i;
909
af6a29bc
TZ		 910 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
911 if (!elt_data)
912 return -ENOMEM;
913
6b4827ad
TZ		 914 for_each_hist_key_field(i, hist_data) {
915 key_field = hist_data->fields[i];
916
917 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
af6a29bc
TZ		 918 elt_data->comm = kzalloc(size, GFP_KERNEL);
919 if (!elt_data->comm) {
920 kfree(elt_data);
6b4827ad 921 return -ENOMEM;
af6a29bc 922 }
6b4827ad
TZ		 923 break;
924 }
925 }
926
af6a29bc
TZ		 927 elt->private_data = elt_data;
928
6b4827ad
TZ		 929 return 0;
930}
931
af6a29bc 932static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
6b4827ad 933{
af6a29bc 934 struct hist_elt_data *elt_data = elt->private_data;
6b4827ad 935
af6a29bc
TZ		 936 if (elt_data->comm)
937 save_comm(elt_data->comm, current);
6b4827ad
TZ		 938}
939
af6a29bc
TZ		 940static const struct tracing_map_ops hist_trigger_elt_data_ops = {
941 .elt_alloc = hist_trigger_elt_data_alloc,
942 .elt_free = hist_trigger_elt_data_free,
943 .elt_init = hist_trigger_elt_data_init,
6b4827ad
TZ		 944};
945
2ece94fb
TZ		 946static const char *get_hist_field_flags(struct hist_field *hist_field)
947{
948 const char *flags_str = NULL;
949
950 if (hist_field->flags & HIST_FIELD_FL_HEX)
951 flags_str = "hex";
952 else if (hist_field->flags & HIST_FIELD_FL_SYM)
953 flags_str = "sym";
954 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
955 flags_str = "sym-offset";
956 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
957 flags_str = "execname";
958 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
959 flags_str = "syscall";
960 else if (hist_field->flags & HIST_FIELD_FL_LOG2)
961 flags_str = "log2";
962 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
963 flags_str = "usecs";
964
965 return flags_str;
966}
967
100719dc
TZ		 968static void expr_field_str(struct hist_field *field, char *expr)
969{
067fe038
TZ		 970 if (field->flags & HIST_FIELD_FL_VAR_REF)
971 strcat(expr, "$");
972
100719dc
TZ		 973 strcat(expr, hist_field_name(field, 0));
974
975 if (field->flags) {
976 const char *flags_str = get_hist_field_flags(field);
977
978 if (flags_str) {
979 strcat(expr, ".");
980 strcat(expr, flags_str);
981 }
982 }
983}
984
985static char *expr_str(struct hist_field *field, unsigned int level)
986{
987 char *expr;
988
989 if (level > 1)
990 return NULL;
991
992 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
993 if (!expr)
994 return NULL;
995
996 if (!field->operands[0]) {
997 expr_field_str(field, expr);
998 return expr;
999 }
1000
1001 if (field->operator == FIELD_OP_UNARY_MINUS) {
1002 char *subexpr;
1003
1004 strcat(expr, "-(");
1005 subexpr = expr_str(field->operands[0], ++level);
1006 if (!subexpr) {
1007 kfree(expr);
1008 return NULL;
1009 }
1010 strcat(expr, subexpr);
1011 strcat(expr, ")");
1012
1013 kfree(subexpr);
1014
1015 return expr;
1016 }
1017
1018 expr_field_str(field->operands[0], expr);
1019
1020 switch (field->operator) {
1021 case FIELD_OP_MINUS:
1022 strcat(expr, "-");
1023 break;
1024 case FIELD_OP_PLUS:
1025 strcat(expr, "+");
1026 break;
1027 default:
1028 kfree(expr);
1029 return NULL;
1030 }
1031
1032 expr_field_str(field->operands[1], expr);
1033
1034 return expr;
1035}
1036
1037static int contains_operator(char *str)
1038{
1039 enum field_op_id field_op = FIELD_OP_NONE;
1040 char *op;
1041
1042 op = strpbrk(str, "+-");
1043 if (!op)
1044 return FIELD_OP_NONE;
1045
1046 switch (*op) {
1047 case '-':
1048 if (*str == '-')
1049 field_op = FIELD_OP_UNARY_MINUS;
1050 else
1051 field_op = FIELD_OP_MINUS;
1052 break;
1053 case '+':
1054 field_op = FIELD_OP_PLUS;
1055 break;
1056 default:
1057 break;
1058 }
1059
1060 return field_op;
1061}
1062
5819eadd
TZ		 1063static void destroy_hist_field(struct hist_field *hist_field,
1064 unsigned int level)
7ef224d1 1065{
5819eadd
TZ		 1066 unsigned int i;
1067
100719dc 1068 if (level > 3)
5819eadd
TZ		 1069 return;
1070
1071 if (!hist_field)
1072 return;
1073
1074 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1075 destroy_hist_field(hist_field->operands[i], level + 1);
1076
30350d65 1077 kfree(hist_field->var.name);
100719dc 1078 kfree(hist_field->name);
19a9facd 1079 kfree(hist_field->type);
30350d65 1080
7ef224d1
TZ		 1081 kfree(hist_field);
1082}
1083
b559d003
TZ		 1084static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1085 struct ftrace_event_field *field,
30350d65
TZ		 1086 unsigned long flags,
1087 char *var_name)
7ef224d1
TZ		 1088{
1089 struct hist_field *hist_field;
1090
1091 if (field && is_function_field(field))
1092 return NULL;
1093
1094 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1095 if (!hist_field)
1096 return NULL;
1097
b559d003
TZ		 1098 hist_field->hist_data = hist_data;
1099
100719dc
TZ		 1100 if (flags & HIST_FIELD_FL_EXPR)
1101 goto out; /* caller will populate */
1102
067fe038
TZ		 1103 if (flags & HIST_FIELD_FL_VAR_REF) {
1104 hist_field->fn = hist_field_var_ref;
1105 goto out;
1106 }
1107
7ef224d1
TZ		 1108 if (flags & HIST_FIELD_FL_HITCOUNT) {
1109 hist_field->fn = hist_field_counter;
19a9facd
TZ		 1110 hist_field->size = sizeof(u64);
1111 hist_field->type = kstrdup("u64", GFP_KERNEL);
1112 if (!hist_field->type)
1113 goto free;
7ef224d1
TZ		 1114 goto out;
1115 }
1116
69a0200c
TZ		 1117 if (flags & HIST_FIELD_FL_STACKTRACE) {
1118 hist_field->fn = hist_field_none;
1119 goto out;
1120 }
1121
4b94f5b7 1122 if (flags & HIST_FIELD_FL_LOG2) {
5819eadd 1123 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
4b94f5b7 1124 hist_field->fn = hist_field_log2;
30350d65 1125 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
5819eadd 1126 hist_field->size = hist_field->operands[0]->size;
19a9facd
TZ		 1127 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1128 if (!hist_field->type)
1129 goto free;
4b94f5b7
NK		 1130 goto out;
1131 }
1132
ad42febe
TZ		 1133 if (flags & HIST_FIELD_FL_TIMESTAMP) {
1134 hist_field->fn = hist_field_timestamp;
1135 hist_field->size = sizeof(u64);
19a9facd
TZ		 1136 hist_field->type = kstrdup("u64", GFP_KERNEL);
1137 if (!hist_field->type)
1138 goto free;
ad42febe
TZ		 1139 goto out;
1140 }
1141
432480c5
TZ		 1142 if (WARN_ON_ONCE(!field))
1143 goto out;
1144
7ef224d1
TZ		 1145 if (is_string_field(field)) {
1146 flags |= HIST_FIELD_FL_STRING;
79e577cb 1147
19a9facd
TZ		 1148 hist_field->size = MAX_FILTER_STR_VAL;
1149 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1150 if (!hist_field->type)
1151 goto free;
1152
79e577cb
NK		 1153 if (field->filter_type == FILTER_STATIC_STRING)
1154 hist_field->fn = hist_field_string;
1155 else if (field->filter_type == FILTER_DYN_STRING)
1156 hist_field->fn = hist_field_dynstring;
1157 else
1158 hist_field->fn = hist_field_pstring;
7ef224d1 1159 } else {
19a9facd
TZ		 1160 hist_field->size = field->size;
1161 hist_field->is_signed = field->is_signed;
1162 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1163 if (!hist_field->type)
1164 goto free;
1165
7ef224d1
TZ		 1166 hist_field->fn = select_value_fn(field->size,
1167 field->is_signed);
1168 if (!hist_field->fn) {
5819eadd 1169 destroy_hist_field(hist_field, 0);
7ef224d1
TZ		 1170 return NULL;
1171 }
1172 }
1173 out:
1174 hist_field->field = field;
1175 hist_field->flags = flags;
1176
30350d65
TZ		 1177 if (var_name) {
1178 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1179 if (!hist_field->var.name)
1180 goto free;
1181 }
1182
7ef224d1 1183 return hist_field;
30350d65
TZ		 1184 free:
1185 destroy_hist_field(hist_field, 0);
1186 return NULL;
7ef224d1
TZ		 1187}
1188
1189static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1190{
1191 unsigned int i;
1192
30350d65 1193 for (i = 0; i < HIST_FIELDS_MAX; i++) {
7ef224d1 1194 if (hist_data->fields[i]) {
5819eadd 1195 destroy_hist_field(hist_data->fields[i], 0);
7ef224d1
TZ		 1196 hist_data->fields[i] = NULL;
1197 }
1198 }
1199}
1200
067fe038
TZ		 1201static int init_var_ref(struct hist_field *ref_field,
1202 struct hist_field *var_field,
1203 char *system, char *event_name)
1204{
1205 int err = 0;
1206
1207 ref_field->var.idx = var_field->var.idx;
1208 ref_field->var.hist_data = var_field->hist_data;
1209 ref_field->size = var_field->size;
1210 ref_field->is_signed = var_field->is_signed;
1211 ref_field->flags |= var_field->flags &
1212 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1213
1214 if (system) {
1215 ref_field->system = kstrdup(system, GFP_KERNEL);
1216 if (!ref_field->system)
1217 return -ENOMEM;
1218 }
1219
1220 if (event_name) {
1221 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1222 if (!ref_field->event_name) {
1223 err = -ENOMEM;
1224 goto free;
1225 }
1226 }
1227
1228 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1229 if (!ref_field->name) {
1230 err = -ENOMEM;
1231 goto free;
1232 }
1233
1234 ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1235 if (!ref_field->type) {
1236 err = -ENOMEM;
1237 goto free;
1238 }
1239 out:
1240 return err;
1241 free:
1242 kfree(ref_field->system);
1243 kfree(ref_field->event_name);
1244 kfree(ref_field->name);
1245
1246 goto out;
1247}
1248
1249static struct hist_field *create_var_ref(struct hist_field *var_field,
1250 char *system, char *event_name)
1251{
1252 unsigned long flags = HIST_FIELD_FL_VAR_REF;
1253 struct hist_field *ref_field;
1254
1255 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1256 if (ref_field) {
1257 if (init_var_ref(ref_field, var_field, system, event_name)) {
1258 destroy_hist_field(ref_field, 0);
1259 return NULL;
1260 }
1261 }
1262
1263 return ref_field;
1264}
1265
1266static bool is_var_ref(char *var_name)
1267{
1268 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1269 return false;
1270
1271 return true;
1272}
1273
1274static char *field_name_from_var(struct hist_trigger_data *hist_data,
1275 char *var_name)
1276{
1277 char *name, *field;
1278 unsigned int i;
1279
1280 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1281 name = hist_data->attrs->var_defs.name[i];
1282
1283 if (strcmp(var_name, name) == 0) {
1284 field = hist_data->attrs->var_defs.expr[i];
1285 if (contains_operator(field) || is_var_ref(field))
1286 continue;
1287 return field;
1288 }
1289 }
1290
1291 return NULL;
1292}
1293
1294static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1295 char *system, char *event_name,
1296 char *var_name)
1297{
1298 struct trace_event_call *call;
1299
1300 if (system && event_name) {
1301 call = hist_data->event_file->event_call;
1302
1303 if (strcmp(system, call->class->system) != 0)
1304 return NULL;
1305
1306 if (strcmp(event_name, trace_event_name(call)) != 0)
1307 return NULL;
1308 }
1309
1310 if (!!system != !!event_name)
1311 return NULL;
1312
1313 if (!is_var_ref(var_name))
1314 return NULL;
1315
1316 var_name++;
1317
1318 return field_name_from_var(hist_data, var_name);
1319}
1320
1321static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1322 char *system, char *event_name,
1323 char *var_name)
1324{
1325 struct hist_field *var_field = NULL, *ref_field = NULL;
1326
1327 if (!is_var_ref(var_name))
1328 return NULL;
1329
1330 var_name++;
1331
1332 var_field = find_event_var(hist_data, system, event_name, var_name);
1333 if (var_field)
1334 ref_field = create_var_ref(var_field, system, event_name);
1335
1336 return ref_field;
1337}
1338
100719dc
TZ		 1339static struct ftrace_event_field *
1340parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1341 char *field_str, unsigned long *flags)
1342{
1343 struct ftrace_event_field *field = NULL;
1344 char *field_name, *modifier, *str;
1345
1346 modifier = str = kstrdup(field_str, GFP_KERNEL);
1347 if (!modifier)
1348 return ERR_PTR(-ENOMEM);
1349
1350 field_name = strsep(&modifier, ".");
1351 if (modifier) {
1352 if (strcmp(modifier, "hex") == 0)
1353 *flags |= HIST_FIELD_FL_HEX;
1354 else if (strcmp(modifier, "sym") == 0)
1355 *flags |= HIST_FIELD_FL_SYM;
1356 else if (strcmp(modifier, "sym-offset") == 0)
1357 *flags |= HIST_FIELD_FL_SYM_OFFSET;
1358 else if ((strcmp(modifier, "execname") == 0) &&
1359 (strcmp(field_name, "common_pid") == 0))
1360 *flags |= HIST_FIELD_FL_EXECNAME;
1361 else if (strcmp(modifier, "syscall") == 0)
1362 *flags |= HIST_FIELD_FL_SYSCALL;
1363 else if (strcmp(modifier, "log2") == 0)
1364 *flags |= HIST_FIELD_FL_LOG2;
1365 else if (strcmp(modifier, "usecs") == 0)
1366 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1367 else {
1368 field = ERR_PTR(-EINVAL);
1369 goto out;
1370 }
1371 }
1372
1373 if (strcmp(field_name, "common_timestamp") == 0) {
1374 *flags |= HIST_FIELD_FL_TIMESTAMP;
1375 hist_data->enable_timestamps = true;
1376 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1377 hist_data->attrs->ts_in_usecs = true;
1378 } else {
1379 field = trace_find_event_field(file->event_call, field_name);
1380 if (!field || !field->size) {
1381 field = ERR_PTR(-EINVAL);
1382 goto out;
1383 }
1384 }
1385 out:
1386 kfree(str);
1387
1388 return field;
1389}
1390
1391static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
1392 struct trace_event_file *file, char *str,
1393 unsigned long *flags, char *var_name)
1394{
067fe038 1395 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
100719dc
TZ		 1396 struct ftrace_event_field *field = NULL;
1397 struct hist_field *hist_field = NULL;
1398 int ret = 0;
1399
067fe038
TZ		 1400 s = strchr(str, '.');
1401 if (s) {
1402 s = strchr(++s, '.');
1403 if (s) {
1404 ref_system = strsep(&str, ".");
1405 if (!str) {
1406 ret = -EINVAL;
1407 goto out;
1408 }
1409 ref_event = strsep(&str, ".");
1410 if (!str) {
1411 ret = -EINVAL;
1412 goto out;
1413 }
1414 ref_var = str;
1415 }
1416 }
1417
1418 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
1419 if (!s) {
1420 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
1421 if (hist_field) {
1422 hist_data->var_refs[hist_data->n_var_refs] = hist_field;
1423 hist_field->var_ref_idx = hist_data->n_var_refs++;
1424 return hist_field;
1425 }
1426 } else
1427 str = s;
1428
100719dc
TZ		 1429 field = parse_field(hist_data, file, str, flags);
1430 if (IS_ERR(field)) {
1431 ret = PTR_ERR(field);
1432 goto out;
1433 }
1434
1435 hist_field = create_hist_field(hist_data, field, *flags, var_name);
1436 if (!hist_field) {
1437 ret = -ENOMEM;
1438 goto out;
1439 }
1440
1441 return hist_field;
1442 out:
1443 return ERR_PTR(ret);
1444}
1445
1446static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
1447 struct trace_event_file *file,
1448 char *str, unsigned long flags,
1449 char *var_name, unsigned int level);
1450
1451static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
1452 struct trace_event_file *file,
1453 char *str, unsigned long flags,
1454 char *var_name, unsigned int level)
1455{
1456 struct hist_field *operand1, *expr = NULL;
1457 unsigned long operand_flags;
1458 int ret = 0;
1459 char *s;
1460
1461 /* we support only -(xxx) i.e. explicit parens required */
1462
1463 if (level > 3) {
1464 ret = -EINVAL;
1465 goto free;
1466 }
1467
1468 str++; /* skip leading '-' */
1469
1470 s = strchr(str, '(');
1471 if (s)
1472 str++;
1473 else {
1474 ret = -EINVAL;
1475 goto free;
1476 }
1477
1478 s = strrchr(str, ')');
1479 if (s)
1480 *s = '\0';
1481 else {
1482 ret = -EINVAL; /* no closing ')' */
1483 goto free;
1484 }
1485
1486 flags |= HIST_FIELD_FL_EXPR;
1487 expr = create_hist_field(hist_data, NULL, flags, var_name);
1488 if (!expr) {
1489 ret = -ENOMEM;
1490 goto free;
1491 }
1492
1493 operand_flags = 0;
1494 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
1495 if (IS_ERR(operand1)) {
1496 ret = PTR_ERR(operand1);
1497 goto free;
1498 }
1499
1500 expr->flags |= operand1->flags &
1501 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1502 expr->fn = hist_field_unary_minus;
1503 expr->operands[0] = operand1;
1504 expr->operator = FIELD_OP_UNARY_MINUS;
1505 expr->name = expr_str(expr, 0);
19a9facd
TZ		 1506 expr->type = kstrdup(operand1->type, GFP_KERNEL);
1507 if (!expr->type) {
1508 ret = -ENOMEM;
1509 goto free;
1510 }
100719dc
TZ		 1511
1512 return expr;
1513 free:
1514 destroy_hist_field(expr, 0);
1515 return ERR_PTR(ret);
1516}
1517
1518static int check_expr_operands(struct hist_field *operand1,
1519 struct hist_field *operand2)
1520{
1521 unsigned long operand1_flags = operand1->flags;
1522 unsigned long operand2_flags = operand2->flags;
1523
1524 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
1525 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS))
1526 return -EINVAL;
1527
1528 return 0;
1529}
1530
1531static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
1532 struct trace_event_file *file,
1533 char *str, unsigned long flags,
1534 char *var_name, unsigned int level)
1535{
1536 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
1537 unsigned long operand_flags;
1538 int field_op, ret = -EINVAL;
1539 char *sep, *operand1_str;
1540
1541 if (level > 3)
1542 return ERR_PTR(-EINVAL);
1543
1544 field_op = contains_operator(str);
1545
1546 if (field_op == FIELD_OP_NONE)
1547 return parse_atom(hist_data, file, str, &flags, var_name);
1548
1549 if (field_op == FIELD_OP_UNARY_MINUS)
1550 return parse_unary(hist_data, file, str, flags, var_name, ++level);
1551
1552 switch (field_op) {
1553 case FIELD_OP_MINUS:
1554 sep = "-";
1555 break;
1556 case FIELD_OP_PLUS:
1557 sep = "+";
1558 break;
1559 default:
1560 goto free;
1561 }
1562
1563 operand1_str = strsep(&str, sep);
1564 if (!operand1_str || !str)
1565 goto free;
1566
1567 operand_flags = 0;
1568 operand1 = parse_atom(hist_data, file, operand1_str,
1569 &operand_flags, NULL);
1570 if (IS_ERR(operand1)) {
1571 ret = PTR_ERR(operand1);
1572 operand1 = NULL;
1573 goto free;
1574 }
1575
1576 /* rest of string could be another expression e.g. b+c in a+b+c */
1577 operand_flags = 0;
1578 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
1579 if (IS_ERR(operand2)) {
1580 ret = PTR_ERR(operand2);
1581 operand2 = NULL;
1582 goto free;
1583 }
1584
1585 ret = check_expr_operands(operand1, operand2);
1586 if (ret)
1587 goto free;
1588
1589 flags |= HIST_FIELD_FL_EXPR;
1590
1591 flags |= operand1->flags &
1592 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1593
1594 expr = create_hist_field(hist_data, NULL, flags, var_name);
1595 if (!expr) {
1596 ret = -ENOMEM;
1597 goto free;
1598 }
1599
067fe038
TZ		 1600 operand1->read_once = true;
1601 operand2->read_once = true;
1602
100719dc
TZ		 1603 expr->operands[0] = operand1;
1604 expr->operands[1] = operand2;
1605 expr->operator = field_op;
1606 expr->name = expr_str(expr, 0);
19a9facd
TZ		 1607 expr->type = kstrdup(operand1->type, GFP_KERNEL);
1608 if (!expr->type) {
1609 ret = -ENOMEM;
1610 goto free;
1611 }
100719dc
TZ		 1612
1613 switch (field_op) {
1614 case FIELD_OP_MINUS:
1615 expr->fn = hist_field_minus;
1616 break;
1617 case FIELD_OP_PLUS:
1618 expr->fn = hist_field_plus;
1619 break;
1620 default:
1621 goto free;
1622 }
1623
1624 return expr;
1625 free:
1626 destroy_hist_field(operand1, 0);
1627 destroy_hist_field(operand2, 0);
1628 destroy_hist_field(expr, 0);
1629
1630 return ERR_PTR(ret);
1631}
1632
7ef224d1
TZ		 1633static int create_hitcount_val(struct hist_trigger_data *hist_data)
1634{
1635 hist_data->fields[HITCOUNT_IDX] =
30350d65 1636 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
7ef224d1
TZ		 1637 if (!hist_data->fields[HITCOUNT_IDX])
1638 return -ENOMEM;
1639
1640 hist_data->n_vals++;
30350d65 1641 hist_data->n_fields++;
7ef224d1
TZ		 1642
1643 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
1644 return -EINVAL;
1645
1646 return 0;
1647}
1648
30350d65
TZ		 1649static int __create_val_field(struct hist_trigger_data *hist_data,
1650 unsigned int val_idx,
1651 struct trace_event_file *file,
1652 char *var_name, char *field_str,
1653 unsigned long flags)
f2606835 1654{
100719dc 1655 struct hist_field *hist_field;
f2606835
TZ		 1656 int ret = 0;
1657
100719dc
TZ		 1658 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
1659 if (IS_ERR(hist_field)) {
1660 ret = PTR_ERR(hist_field);
f2606835
TZ		 1661 goto out;
1662 }
1663
100719dc
TZ		 1664 hist_data->fields[val_idx] = hist_field;
1665
f2606835 1666 ++hist_data->n_vals;
30350d65 1667 ++hist_data->n_fields;
f2606835 1668
30350d65 1669 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
f2606835
TZ		 1670 ret = -EINVAL;
1671 out:
1672 return ret;
1673}
1674
30350d65
TZ		 1675static int create_val_field(struct hist_trigger_data *hist_data,
1676 unsigned int val_idx,
1677 struct trace_event_file *file,
1678 char *field_str)
1679{
1680 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
1681 return -EINVAL;
1682
1683 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
1684}
1685
1686static int create_var_field(struct hist_trigger_data *hist_data,
1687 unsigned int val_idx,
1688 struct trace_event_file *file,
1689 char *var_name, char *expr_str)
1690{
1691 unsigned long flags = 0;
1692
1693 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
1694 return -EINVAL;
1695 if (find_var(hist_data, file, var_name) && !hist_data->remove) {
1696 return -EINVAL;
1697 }
1698
1699 flags |= HIST_FIELD_FL_VAR;
1700 hist_data->n_vars++;
1701 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
1702 return -EINVAL;
1703
1704 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
1705}
1706
7ef224d1
TZ		 1707static int create_val_fields(struct hist_trigger_data *hist_data,
1708 struct trace_event_file *file)
1709{
f2606835 1710 char *fields_str, *field_str;
30350d65 1711 unsigned int i, j = 1;
7ef224d1
TZ		 1712 int ret;
1713
1714 ret = create_hitcount_val(hist_data);
f2606835
TZ		 1715 if (ret)
1716 goto out;
7ef224d1 1717
f2606835
TZ		 1718 fields_str = hist_data->attrs->vals_str;
1719 if (!fields_str)
1720 goto out;
1721
1722 strsep(&fields_str, "=");
1723 if (!fields_str)
1724 goto out;
1725
1726 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
1727 j < TRACING_MAP_VALS_MAX; i++) {
1728 field_str = strsep(&fields_str, ",");
1729 if (!field_str)
1730 break;
30350d65 1731
f2606835
TZ		 1732 if (strcmp(field_str, "hitcount") == 0)
1733 continue;
30350d65 1734
f2606835
TZ		 1735 ret = create_val_field(hist_data, j++, file, field_str);
1736 if (ret)
1737 goto out;
1738 }
30350d65 1739
f2606835
TZ		 1740 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
1741 ret = -EINVAL;
1742 out:
7ef224d1
TZ		 1743 return ret;
1744}
1745
1746static int create_key_field(struct hist_trigger_data *hist_data,
1747 unsigned int key_idx,
76a3b0c8 1748 unsigned int key_offset,
7ef224d1
TZ		 1749 struct trace_event_file *file,
1750 char *field_str)
1751{
30350d65 1752 struct hist_field *hist_field = NULL;
100719dc 1753
7ef224d1
TZ		 1754 unsigned long flags = 0;
1755 unsigned int key_size;
1756 int ret = 0;
1757
30350d65 1758 if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
7ef224d1
TZ		 1759 return -EINVAL;
1760
1761 flags |= HIST_FIELD_FL_KEY;
1762
69a0200c
TZ		 1763 if (strcmp(field_str, "stacktrace") == 0) {
1764 flags |= HIST_FIELD_FL_STACKTRACE;
1765 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
30350d65 1766 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
69a0200c 1767 } else {
100719dc
TZ		 1768 hist_field = parse_expr(hist_data, file, field_str, flags,
1769 NULL, 0);
1770 if (IS_ERR(hist_field)) {
1771 ret = PTR_ERR(hist_field);
1772 goto out;
69a0200c
TZ		 1773 }
1774
067fe038
TZ		 1775 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
1776 destroy_hist_field(hist_field, 0);
1777 ret = -EINVAL;
1778 goto out;
1779 }
1780
100719dc 1781 key_size = hist_field->size;
7ef224d1
TZ		 1782 }
1783
100719dc 1784 hist_data->fields[key_idx] = hist_field;
7ef224d1
TZ		 1785
1786 key_size = ALIGN(key_size, sizeof(u64));
1787 hist_data->fields[key_idx]->size = key_size;
76a3b0c8 1788 hist_data->fields[key_idx]->offset = key_offset;
100719dc 1789
76a3b0c8 1790 hist_data->key_size += key_size;
100719dc 1791
7ef224d1
TZ		 1792 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
1793 ret = -EINVAL;
1794 goto out;
1795 }
1796
1797 hist_data->n_keys++;
30350d65 1798 hist_data->n_fields++;
7ef224d1
TZ		 1799
1800 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
1801 return -EINVAL;
1802
1803 ret = key_size;
1804 out:
1805 return ret;
1806}
1807
1808static int create_key_fields(struct hist_trigger_data *hist_data,
1809 struct trace_event_file *file)
1810{
76a3b0c8 1811 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
7ef224d1
TZ		 1812 char *fields_str, *field_str;
1813 int ret = -EINVAL;
1814
1815 fields_str = hist_data->attrs->keys_str;
1816 if (!fields_str)
1817 goto out;
1818
1819 strsep(&fields_str, "=");
1820 if (!fields_str)
1821 goto out;
1822
76a3b0c8 1823 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
7ef224d1
TZ		 1824 field_str = strsep(&fields_str, ",");
1825 if (!field_str)
1826 break;
76a3b0c8
TZ		 1827 ret = create_key_field(hist_data, i, key_offset,
1828 file, field_str);
7ef224d1
TZ		 1829 if (ret < 0)
1830 goto out;
76a3b0c8 1831 key_offset += ret;
7ef224d1
TZ		 1832 }
1833 if (fields_str) {
1834 ret = -EINVAL;
1835 goto out;
1836 }
1837 ret = 0;
1838 out:
1839 return ret;
1840}
1841
30350d65
TZ		 1842static int create_var_fields(struct hist_trigger_data *hist_data,
1843 struct trace_event_file *file)
1844{
1845 unsigned int i, j = hist_data->n_vals;
1846 int ret = 0;
1847
1848 unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
1849
1850 for (i = 0; i < n_vars; i++) {
1851 char *var_name = hist_data->attrs->var_defs.name[i];
1852 char *expr = hist_data->attrs->var_defs.expr[i];
1853
1854 ret = create_var_field(hist_data, j++, file, var_name, expr);
1855 if (ret)
1856 goto out;
1857 }
1858 out:
1859 return ret;
1860}
1861
1862static void free_var_defs(struct hist_trigger_data *hist_data)
1863{
1864 unsigned int i;
1865
1866 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1867 kfree(hist_data->attrs->var_defs.name[i]);
1868 kfree(hist_data->attrs->var_defs.expr[i]);
1869 }
1870
1871 hist_data->attrs->var_defs.n_vars = 0;
1872}
1873
1874static int parse_var_defs(struct hist_trigger_data *hist_data)
1875{
1876 char *s, *str, *var_name, *field_str;
1877 unsigned int i, j, n_vars = 0;
1878 int ret = 0;
1879
1880 for (i = 0; i < hist_data->attrs->n_assignments; i++) {
1881 str = hist_data->attrs->assignment_str[i];
1882 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
1883 field_str = strsep(&str, ",");
1884 if (!field_str)
1885 break;
1886
1887 var_name = strsep(&field_str, "=");
1888 if (!var_name || !field_str) {
1889 ret = -EINVAL;
1890 goto free;
1891 }
1892
1893 if (n_vars == TRACING_MAP_VARS_MAX) {
1894 ret = -EINVAL;
1895 goto free;
1896 }
1897
1898 s = kstrdup(var_name, GFP_KERNEL);
1899 if (!s) {
1900 ret = -ENOMEM;
1901 goto free;
1902 }
1903 hist_data->attrs->var_defs.name[n_vars] = s;
1904
1905 s = kstrdup(field_str, GFP_KERNEL);
1906 if (!s) {
1907 kfree(hist_data->attrs->var_defs.name[n_vars]);
1908 ret = -ENOMEM;
1909 goto free;
1910 }
1911 hist_data->attrs->var_defs.expr[n_vars++] = s;
1912
1913 hist_data->attrs->var_defs.n_vars = n_vars;
1914 }
1915 }
1916
1917 return ret;
1918 free:
1919 free_var_defs(hist_data);
1920
1921 return ret;
1922}
1923
7ef224d1
TZ		 1924static int create_hist_fields(struct hist_trigger_data *hist_data,
1925 struct trace_event_file *file)
1926{
1927 int ret;
1928
30350d65
TZ		 1929 ret = parse_var_defs(hist_data);
1930 if (ret)
1931 goto out;
1932
7ef224d1
TZ		 1933 ret = create_val_fields(hist_data, file);
1934 if (ret)
1935 goto out;
1936
30350d65 1937 ret = create_var_fields(hist_data, file);
7ef224d1
TZ		 1938 if (ret)
1939 goto out;
1940
30350d65
TZ		 1941 ret = create_key_fields(hist_data, file);
1942 if (ret)
1943 goto out;
7ef224d1 1944 out:
30350d65
TZ		 1945 free_var_defs(hist_data);
1946
7ef224d1
TZ		 1947 return ret;
1948}
1949
e62347d2
TZ		 1950static int is_descending(const char *str)
1951{
1952 if (!str)
1953 return 0;
1954
1955 if (strcmp(str, "descending") == 0)
1956 return 1;
1957
1958 if (strcmp(str, "ascending") == 0)
1959 return 0;
1960
1961 return -EINVAL;
1962}
1963
7ef224d1
TZ		 1964static int create_sort_keys(struct hist_trigger_data *hist_data)
1965{
e62347d2 1966 char *fields_str = hist_data->attrs->sort_key_str;
e62347d2
TZ		 1967 struct tracing_map_sort_key *sort_key;
1968 int descending, ret = 0;
30350d65 1969 unsigned int i, j, k;
e62347d2
TZ		 1970
1971 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
1972
1973 if (!fields_str)
1974 goto out;
1975
1976 strsep(&fields_str, "=");
1977 if (!fields_str) {
1978 ret = -EINVAL;
1979 goto out;
1980 }
1981
1982 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
85013256 1983 struct hist_field *hist_field;
e62347d2 1984 char *field_str, *field_name;
85013256 1985 const char *test_name;
e62347d2
TZ		 1986
1987 sort_key = &hist_data->sort_keys[i];
1988
1989 field_str = strsep(&fields_str, ",");
1990 if (!field_str) {
1991 if (i == 0)
1992 ret = -EINVAL;
1993 break;
1994 }
1995
1996 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
1997 ret = -EINVAL;
1998 break;
1999 }
7ef224d1 2000
e62347d2
TZ		 2001 field_name = strsep(&field_str, ".");
2002 if (!field_name) {
2003 ret = -EINVAL;
2004 break;
2005 }
2006
2007 if (strcmp(field_name, "hitcount") == 0) {
2008 descending = is_descending(field_str);
2009 if (descending < 0) {
2010 ret = descending;
2011 break;
2012 }
2013 sort_key->descending = descending;
2014 continue;
2015 }
7ef224d1 2016
30350d65
TZ		 2017 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
2018 unsigned int idx;
2019
85013256 2020 hist_field = hist_data->fields[j];
30350d65
TZ		 2021 if (hist_field->flags & HIST_FIELD_FL_VAR)
2022 continue;
2023
2024 idx = k++;
2025
85013256
TZ		 2026 test_name = hist_field_name(hist_field, 0);
2027
2028 if (strcmp(field_name, test_name) == 0) {
30350d65 2029 sort_key->field_idx = idx;
e62347d2
TZ		 2030 descending = is_descending(field_str);
2031 if (descending < 0) {
2032 ret = descending;
2033 goto out;
2034 }
2035 sort_key->descending = descending;
2036 break;
2037 }
2038 }
2039 if (j == hist_data->n_fields) {
2040 ret = -EINVAL;
2041 break;
2042 }
2043 }
30350d65 2044
e62347d2
TZ		 2045 hist_data->n_sort_keys = i;
2046 out:
7ef224d1
TZ		 2047 return ret;
2048}
2049
2050static void destroy_hist_data(struct hist_trigger_data *hist_data)
2051{
2052 destroy_hist_trigger_attrs(hist_data->attrs);
2053 destroy_hist_fields(hist_data);
2054 tracing_map_destroy(hist_data->map);
2055 kfree(hist_data);
2056}
2057
2058static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
2059{
2060 struct tracing_map *map = hist_data->map;
2061 struct ftrace_event_field *field;
2062 struct hist_field *hist_field;
d50c744e 2063 int i, idx;
7ef224d1
TZ		 2064
2065 for_each_hist_field(i, hist_data) {
2066 hist_field = hist_data->fields[i];
2067 if (hist_field->flags & HIST_FIELD_FL_KEY) {
2068 tracing_map_cmp_fn_t cmp_fn;
2069
2070 field = hist_field->field;
2071
69a0200c
TZ		 2072 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
2073 cmp_fn = tracing_map_cmp_none;
ad42febe
TZ		 2074 else if (!field)
2075 cmp_fn = tracing_map_cmp_num(hist_field->size,
2076 hist_field->is_signed);
69a0200c 2077 else if (is_string_field(field))
7ef224d1
TZ		 2078 cmp_fn = tracing_map_cmp_string;
2079 else
2080 cmp_fn = tracing_map_cmp_num(field->size,
2081 field->is_signed);
76a3b0c8
TZ		 2082 idx = tracing_map_add_key_field(map,
2083 hist_field->offset,
2084 cmp_fn);
30350d65 2085 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
7ef224d1
TZ		 2086 idx = tracing_map_add_sum_field(map);
2087
2088 if (idx < 0)
2089 return idx;
30350d65
TZ		 2090
2091 if (hist_field->flags & HIST_FIELD_FL_VAR) {
2092 idx = tracing_map_add_var(map);
2093 if (idx < 0)
2094 return idx;
2095 hist_field->var.idx = idx;
2096 hist_field->var.hist_data = hist_data;
2097 }
7ef224d1
TZ		 2098 }
2099
2100 return 0;
2101}
2102
2103static struct hist_trigger_data *
2104create_hist_data(unsigned int map_bits,
2105 struct hist_trigger_attrs *attrs,
30350d65
TZ		 2106 struct trace_event_file *file,
2107 bool remove)
7ef224d1 2108{
6b4827ad 2109 const struct tracing_map_ops *map_ops = NULL;
7ef224d1
TZ		 2110 struct hist_trigger_data *hist_data;
2111 int ret = 0;
2112
2113 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
2114 if (!hist_data)
2115 return ERR_PTR(-ENOMEM);
2116
2117 hist_data->attrs = attrs;
30350d65 2118 hist_data->remove = remove;
067fe038 2119 hist_data->event_file = file;
7ef224d1
TZ		 2120
2121 ret = create_hist_fields(hist_data, file);
2122 if (ret)
2123 goto free;
2124
2125 ret = create_sort_keys(hist_data);
2126 if (ret)
2127 goto free;
2128
af6a29bc 2129 map_ops = &hist_trigger_elt_data_ops;
6b4827ad 2130
7ef224d1 2131 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
6b4827ad 2132 map_ops, hist_data);
7ef224d1
TZ		 2133 if (IS_ERR(hist_data->map)) {
2134 ret = PTR_ERR(hist_data->map);
2135 hist_data->map = NULL;
2136 goto free;
2137 }
2138
2139 ret = create_tracing_map_fields(hist_data);
2140 if (ret)
2141 goto free;
7ef224d1
TZ		 2142 out:
2143 return hist_data;
2144 free:
2145 hist_data->attrs = NULL;
2146
2147 destroy_hist_data(hist_data);
2148
2149 hist_data = ERR_PTR(ret);
2150
2151 goto out;
2152}
2153
2154static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
fbd302cb 2155 struct tracing_map_elt *elt, void *rec,
067fe038
TZ		 2156 struct ring_buffer_event *rbe,
2157 u64 *var_ref_vals)
7ef224d1 2158{
067fe038 2159 struct hist_elt_data *elt_data;
7ef224d1 2160 struct hist_field *hist_field;
30350d65 2161 unsigned int i, var_idx;
7ef224d1
TZ		 2162 u64 hist_val;
2163
067fe038
TZ		 2164 elt_data = elt->private_data;
2165 elt_data->var_ref_vals = var_ref_vals;
2166
7ef224d1
TZ		 2167 for_each_hist_val_field(i, hist_data) {
2168 hist_field = hist_data->fields[i];
df35d93b 2169 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
30350d65
TZ		 2170 if (hist_field->flags & HIST_FIELD_FL_VAR) {
2171 var_idx = hist_field->var.idx;
2172 tracing_map_set_var(elt, var_idx, hist_val);
2173 continue;
2174 }
7ef224d1
TZ		 2175 tracing_map_update_sum(elt, i, hist_val);
2176 }
30350d65
TZ		 2177
2178 for_each_hist_key_field(i, hist_data) {
2179 hist_field = hist_data->fields[i];
2180 if (hist_field->flags & HIST_FIELD_FL_VAR) {
df35d93b 2181 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
30350d65
TZ		 2182 var_idx = hist_field->var.idx;
2183 tracing_map_set_var(elt, var_idx, hist_val);
2184 }
2185 }
7ef224d1
TZ		 2186}
2187
6a475cb1
TZ		 2188static inline void add_to_key(char *compound_key, void *key,
2189 struct hist_field *key_field, void *rec)
2190{
2191 size_t size = key_field->size;
2192
2193 if (key_field->flags & HIST_FIELD_FL_STRING) {
2194 struct ftrace_event_field *field;
2195
2196 field = key_field->field;
2197 if (field->filter_type == FILTER_DYN_STRING)
2198 size = *(u32 *)(rec + field->offset) >> 16;
2199 else if (field->filter_type == FILTER_PTR_STRING)
2200 size = strlen(key);
2201 else if (field->filter_type == FILTER_STATIC_STRING)
2202 size = field->size;
2203
2204 /* ensure NULL-termination */
2205 if (size > key_field->size - 1)
2206 size = key_field->size - 1;
2207 }
2208
2209 memcpy(compound_key + key_field->offset, key, size);
2210}
2211
1ac4f51c 2212static void event_hist_trigger(struct event_trigger_data *data, void *rec,
fbd302cb 2213 struct ring_buffer_event *rbe)
7ef224d1
TZ		 2214{
2215 struct hist_trigger_data *hist_data = data->private_data;
6a475cb1 2216 bool use_compound_key = (hist_data->n_keys > 1);
69a0200c 2217 unsigned long entries[HIST_STACKTRACE_DEPTH];
067fe038 2218 u64 var_ref_vals[TRACING_MAP_VARS_MAX];
76a3b0c8 2219 char compound_key[HIST_KEY_SIZE_MAX];
df35d93b 2220 struct tracing_map_elt *elt = NULL;
69a0200c 2221 struct stack_trace stacktrace;
7ef224d1 2222 struct hist_field *key_field;
7ef224d1
TZ		 2223 u64 field_contents;
2224 void *key = NULL;
2225 unsigned int i;
2226
6a475cb1 2227 memset(compound_key, 0, hist_data->key_size);
76a3b0c8 2228
7ef224d1
TZ		 2229 for_each_hist_key_field(i, hist_data) {
2230 key_field = hist_data->fields[i];
2231
69a0200c
TZ		 2232 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
2233 stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
2234 stacktrace.entries = entries;
2235 stacktrace.nr_entries = 0;
2236 stacktrace.skip = HIST_STACKTRACE_SKIP;
76a3b0c8 2237
69a0200c
TZ		 2238 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
2239 save_stack_trace(&stacktrace);
2240
2241 key = entries;
2242 } else {
df35d93b 2243 field_contents = key_field->fn(key_field, elt, rbe, rec);
6a475cb1 2244 if (key_field->flags & HIST_FIELD_FL_STRING) {
69a0200c 2245 key = (void *)(unsigned long)field_contents;
6a475cb1
TZ		 2246 use_compound_key = true;
2247 } else
69a0200c 2248 key = (void *)&field_contents;
76a3b0c8 2249 }
6a475cb1
TZ		 2250
2251 if (use_compound_key)
2252 add_to_key(compound_key, key, key_field, rec);
7ef224d1
TZ		 2253 }
2254
6a475cb1 2255 if (use_compound_key)
76a3b0c8
TZ		 2256 key = compound_key;
2257
067fe038
TZ		 2258 if (hist_data->n_var_refs &&
2259 !resolve_var_refs(hist_data, key, var_ref_vals, false))
2260 return;
2261
7ef224d1 2262 elt = tracing_map_insert(hist_data->map, key);
067fe038
TZ		 2263 if (!elt)
2264 return;
2265
2266 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
7ef224d1
TZ		 2267}
2268
69a0200c
TZ		 2269static void hist_trigger_stacktrace_print(struct seq_file *m,
2270 unsigned long *stacktrace_entries,
2271 unsigned int max_entries)
2272{
2273 char str[KSYM_SYMBOL_LEN];
2274 unsigned int spaces = 8;
2275 unsigned int i;
2276
2277 for (i = 0; i < max_entries; i++) {
2278 if (stacktrace_entries[i] == ULONG_MAX)
2279 return;
2280
2281 seq_printf(m, "%*c", 1 + spaces, ' ');
2282 sprint_symbol(str, stacktrace_entries[i]);
2283 seq_printf(m, "%s\n", str);
2284 }
2285}
2286
7ef224d1
TZ		 2287static void
2288hist_trigger_entry_print(struct seq_file *m,
2289 struct hist_trigger_data *hist_data, void *key,
2290 struct tracing_map_elt *elt)
2291{
2292 struct hist_field *key_field;
c6afad49 2293 char str[KSYM_SYMBOL_LEN];
69a0200c 2294 bool multiline = false;
85013256 2295 const char *field_name;
7ef224d1
TZ		 2296 unsigned int i;
2297 u64 uval;
2298
2299 seq_puts(m, "{ ");
2300
2301 for_each_hist_key_field(i, hist_data) {
2302 key_field = hist_data->fields[i];
2303
2304 if (i > hist_data->n_vals)
2305 seq_puts(m, ", ");
2306
85013256
TZ		 2307 field_name = hist_field_name(key_field, 0);
2308
0c4a6b46
TZ		 2309 if (key_field->flags & HIST_FIELD_FL_HEX) {
2310 uval = *(u64 *)(key + key_field->offset);
85013256 2311 seq_printf(m, "%s: %llx", field_name, uval);
c6afad49
TZ		 2312 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
2313 uval = *(u64 *)(key + key_field->offset);
2314 sprint_symbol_no_offset(str, uval);
85013256
TZ		 2315 seq_printf(m, "%s: [%llx] %-45s", field_name,
2316 uval, str);
c6afad49
TZ		 2317 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
2318 uval = *(u64 *)(key + key_field->offset);
2319 sprint_symbol(str, uval);
85013256
TZ		 2320 seq_printf(m, "%s: [%llx] %-55s", field_name,
2321 uval, str);
6b4827ad 2322 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
af6a29bc
TZ		 2323 struct hist_elt_data *elt_data = elt->private_data;
2324 char *comm;
2325
2326 if (WARN_ON_ONCE(!elt_data))
2327 return;
2328
2329 comm = elt_data->comm;
6b4827ad
TZ		 2330
2331 uval = *(u64 *)(key + key_field->offset);
85013256
TZ		 2332 seq_printf(m, "%s: %-16s[%10llu]", field_name,
2333 comm, uval);
31696198
TZ		 2334 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
2335 const char *syscall_name;
2336
2337 uval = *(u64 *)(key + key_field->offset);
2338 syscall_name = get_syscall_name(uval);
2339 if (!syscall_name)
2340 syscall_name = "unknown_syscall";
2341
85013256
TZ		 2342 seq_printf(m, "%s: %-30s[%3llu]", field_name,
2343 syscall_name, uval);
69a0200c
TZ		 2344 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
2345 seq_puts(m, "stacktrace:\n");
2346 hist_trigger_stacktrace_print(m,
2347 key + key_field->offset,
2348 HIST_STACKTRACE_DEPTH);
2349 multiline = true;
4b94f5b7 2350 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
85013256 2351 seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4b94f5b7 2352 *(u64 *)(key + key_field->offset));
0c4a6b46 2353 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
85013256 2354 seq_printf(m, "%s: %-50s", field_name,
76a3b0c8 2355 (char *)(key + key_field->offset));
7ef224d1 2356 } else {
76a3b0c8 2357 uval = *(u64 *)(key + key_field->offset);
85013256 2358 seq_printf(m, "%s: %10llu", field_name, uval);
7ef224d1
TZ		 2359 }
2360 }
2361
69a0200c
TZ		 2362 if (!multiline)
2363 seq_puts(m, " ");
2364
2365 seq_puts(m, "}");
7ef224d1
TZ		 2366
2367 seq_printf(m, " hitcount: %10llu",
2368 tracing_map_read_sum(elt, HITCOUNT_IDX));
2369
f2606835 2370 for (i = 1; i < hist_data->n_vals; i++) {
85013256
TZ		 2371 field_name = hist_field_name(hist_data->fields[i], 0);
2372
100719dc
TZ		 2373 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
2374 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
30350d65
TZ		 2375 continue;
2376
0c4a6b46 2377 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
85013256 2378 seq_printf(m, " %s: %10llx", field_name,
0c4a6b46
TZ		 2379 tracing_map_read_sum(elt, i));
2380 } else {
85013256 2381 seq_printf(m, " %s: %10llu", field_name,
0c4a6b46
TZ		 2382 tracing_map_read_sum(elt, i));
2383 }
f2606835
TZ		 2384 }
2385
7ef224d1
TZ		 2386 seq_puts(m, "\n");
2387}
2388
2389static int print_entries(struct seq_file *m,
2390 struct hist_trigger_data *hist_data)
2391{
2392 struct tracing_map_sort_entry **sort_entries = NULL;
2393 struct tracing_map *map = hist_data->map;
d50c744e 2394 int i, n_entries;
7ef224d1
TZ		 2395
2396 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
2397 hist_data->n_sort_keys,
2398 &sort_entries);
2399 if (n_entries < 0)
2400 return n_entries;
2401
2402 for (i = 0; i < n_entries; i++)
2403 hist_trigger_entry_print(m, hist_data,
2404 sort_entries[i]->key,
2405 sort_entries[i]->elt);
2406
2407 tracing_map_destroy_sort_entries(sort_entries, n_entries);
2408
2409 return n_entries;
2410}
2411
52a7f16d
TZ		 2412static void hist_trigger_show(struct seq_file *m,
2413 struct event_trigger_data *data, int n)
7ef224d1 2414{
7ef224d1 2415 struct hist_trigger_data *hist_data;
6e7a2398 2416 int n_entries;
7ef224d1 2417
52a7f16d
TZ		 2418 if (n > 0)
2419 seq_puts(m, "\n\n");
7ef224d1
TZ		 2420
2421 seq_puts(m, "# event histogram\n#\n# trigger info: ");
2422 data->ops->print(m, data->ops, data);
52a7f16d 2423 seq_puts(m, "#\n\n");
7ef224d1
TZ		 2424
2425 hist_data = data->private_data;
2426 n_entries = print_entries(m, hist_data);
6e7a2398 2427 if (n_entries < 0)
7ef224d1 2428 n_entries = 0;
7ef224d1
TZ		 2429
2430 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
2431 (u64)atomic64_read(&hist_data->map->hits),
2432 n_entries, (u64)atomic64_read(&hist_data->map->drops));
52a7f16d
TZ		 2433}
2434
2435static int hist_show(struct seq_file *m, void *v)
2436{
2437 struct event_trigger_data *data;
2438 struct trace_event_file *event_file;
2439 int n = 0, ret = 0;
2440
2441 mutex_lock(&event_mutex);
2442
2443 event_file = event_file_data(m->private);
2444 if (unlikely(!event_file)) {
2445 ret = -ENODEV;
2446 goto out_unlock;
2447 }
2448
2449 list_for_each_entry_rcu(data, &event_file->triggers, list) {
2450 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
2451 hist_trigger_show(m, data, n++);
2452 }
2453
7ef224d1
TZ		 2454 out_unlock:
2455 mutex_unlock(&event_mutex);
2456
2457 return ret;
2458}
2459
2460static int event_hist_open(struct inode *inode, struct file *file)
2461{
2462 return single_open(file, hist_show, file);
2463}
2464
2465const struct file_operations event_hist_fops = {
2466 .open = event_hist_open,
2467 .read = seq_read,
2468 .llseek = seq_lseek,
2469 .release = single_release,
2470};
2471
2472static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
2473{
85013256
TZ		 2474 const char *field_name = hist_field_name(hist_field, 0);
2475
30350d65
TZ		 2476 if (hist_field->var.name)
2477 seq_printf(m, "%s=", hist_field->var.name);
2478
ad42febe
TZ		 2479 if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
2480 seq_puts(m, "common_timestamp");
067fe038
TZ		 2481 else if (field_name) {
2482 if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2483 seq_putc(m, '$');
ad42febe 2484 seq_printf(m, "%s", field_name);
067fe038 2485 }
ad42febe 2486
0c4a6b46
TZ		 2487 if (hist_field->flags) {
2488 const char *flags_str = get_hist_field_flags(hist_field);
2489
2490 if (flags_str)
2491 seq_printf(m, ".%s", flags_str);
2492 }
7ef224d1
TZ		 2493}
2494
2495static int event_hist_trigger_print(struct seq_file *m,
2496 struct event_trigger_ops *ops,
2497 struct event_trigger_data *data)
2498{
2499 struct hist_trigger_data *hist_data = data->private_data;
30350d65
TZ		 2500 struct hist_field *field;
2501 bool have_var = false;
7ef224d1
TZ		 2502 unsigned int i;
2503
5463bfda
TZ		 2504 seq_puts(m, "hist:");
2505
2506 if (data->name)
2507 seq_printf(m, "%s:", data->name);
2508
2509 seq_puts(m, "keys=");
7ef224d1
TZ		 2510
2511 for_each_hist_key_field(i, hist_data) {
30350d65 2512 field = hist_data->fields[i];
7ef224d1
TZ		 2513
2514 if (i > hist_data->n_vals)
2515 seq_puts(m, ",");
2516
30350d65 2517 if (field->flags & HIST_FIELD_FL_STACKTRACE)
69a0200c
TZ		 2518 seq_puts(m, "stacktrace");
2519 else
30350d65 2520 hist_field_print(m, field);
7ef224d1
TZ		 2521 }
2522
2523 seq_puts(m, ":vals=");
f2606835
TZ		 2524
2525 for_each_hist_val_field(i, hist_data) {
30350d65
TZ		 2526 field = hist_data->fields[i];
2527 if (field->flags & HIST_FIELD_FL_VAR) {
2528 have_var = true;
2529 continue;
2530 }
2531
f2606835
TZ		 2532 if (i == HITCOUNT_IDX)
2533 seq_puts(m, "hitcount");
2534 else {
2535 seq_puts(m, ",");
30350d65
TZ		 2536 hist_field_print(m, field);
2537 }
2538 }
2539
2540 if (have_var) {
2541 unsigned int n = 0;
2542
2543 seq_puts(m, ":");
2544
2545 for_each_hist_val_field(i, hist_data) {
2546 field = hist_data->fields[i];
2547
2548 if (field->flags & HIST_FIELD_FL_VAR) {
2549 if (n++)
2550 seq_puts(m, ",");
2551 hist_field_print(m, field);
2552 }
f2606835
TZ		 2553 }
2554 }
7ef224d1
TZ		 2555
2556 seq_puts(m, ":sort=");
e62347d2
TZ		 2557
2558 for (i = 0; i < hist_data->n_sort_keys; i++) {
2559 struct tracing_map_sort_key *sort_key;
30350d65
TZ		 2560 unsigned int idx, first_key_idx;
2561
2562 /* skip VAR vals */
2563 first_key_idx = hist_data->n_vals - hist_data->n_vars;
e62347d2
TZ		 2564
2565 sort_key = &hist_data->sort_keys[i];
ad42febe
TZ		 2566 idx = sort_key->field_idx;
2567
1a361dfc 2568 if (WARN_ON(idx >= HIST_FIELDS_MAX))
ad42febe 2569 return -EINVAL;
e62347d2
TZ		 2570
2571 if (i > 0)
2572 seq_puts(m, ",");
2573
ad42febe 2574 if (idx == HITCOUNT_IDX)
e62347d2 2575 seq_puts(m, "hitcount");
30350d65
TZ		 2576 else {
2577 if (idx >= first_key_idx)
2578 idx += hist_data->n_vars;
e62347d2 2579 hist_field_print(m, hist_data->fields[idx]);
30350d65 2580 }
e62347d2
TZ		 2581
2582 if (sort_key->descending)
2583 seq_puts(m, ".descending");
2584 }
7ef224d1
TZ		 2585 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
2586
2587 if (data->filter_str)
2588 seq_printf(m, " if %s", data->filter_str);
2589
83e99914
TZ		 2590 if (data->paused)
2591 seq_puts(m, " [paused]");
2592 else
2593 seq_puts(m, " [active]");
7ef224d1
TZ		 2594
2595 seq_putc(m, '\n');
2596
2597 return 0;
2598}
2599
5463bfda
TZ		 2600static int event_hist_trigger_init(struct event_trigger_ops *ops,
2601 struct event_trigger_data *data)
2602{
2603 struct hist_trigger_data *hist_data = data->private_data;
2604
2605 if (!data->ref && hist_data->attrs->name)
2606 save_named_trigger(hist_data->attrs->name, data);
2607
2608 data->ref++;
2609
2610 return 0;
2611}
2612
7ef224d1
TZ		 2613static void event_hist_trigger_free(struct event_trigger_ops *ops,
2614 struct event_trigger_data *data)
2615{
2616 struct hist_trigger_data *hist_data = data->private_data;
2617
2618 if (WARN_ON_ONCE(data->ref <= 0))
2619 return;
2620
2621 data->ref--;
2622 if (!data->ref) {
5463bfda
TZ		 2623 if (data->name)
2624 del_named_trigger(data);
067fe038 2625
7ef224d1 2626 trigger_data_free(data);
067fe038
TZ		 2627
2628 remove_hist_vars(hist_data);
2629
7ef224d1
TZ		 2630 destroy_hist_data(hist_data);
2631 }
2632}
2633
2634static struct event_trigger_ops event_hist_trigger_ops = {
2635 .func = event_hist_trigger,
2636 .print = event_hist_trigger_print,
5463bfda 2637 .init = event_hist_trigger_init,
7ef224d1
TZ		 2638 .free = event_hist_trigger_free,
2639};
2640
5463bfda
TZ		 2641static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
2642 struct event_trigger_data *data)
2643{
2644 data->ref++;
2645
2646 save_named_trigger(data->named_data->name, data);
2647
2648 event_hist_trigger_init(ops, data->named_data);
2649
2650 return 0;
2651}
2652
2653static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
2654 struct event_trigger_data *data)
2655{
2656 if (WARN_ON_ONCE(data->ref <= 0))
2657 return;
2658
2659 event_hist_trigger_free(ops, data->named_data);
2660
2661 data->ref--;
2662 if (!data->ref) {
2663 del_named_trigger(data);
2664 trigger_data_free(data);
2665 }
2666}
2667
2668static struct event_trigger_ops event_hist_trigger_named_ops = {
2669 .func = event_hist_trigger,
2670 .print = event_hist_trigger_print,
2671 .init = event_hist_trigger_named_init,
2672 .free = event_hist_trigger_named_free,
2673};
2674
7ef224d1
TZ		 2675static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
2676 char *param)
2677{
2678 return &event_hist_trigger_ops;
2679}
2680
e86ae9ba
TZ		 2681static void hist_clear(struct event_trigger_data *data)
2682{
2683 struct hist_trigger_data *hist_data = data->private_data;
e86ae9ba 2684
5463bfda
TZ		 2685 if (data->name)
2686 pause_named_trigger(data);
e86ae9ba
TZ		 2687
2688 synchronize_sched();
2689
2690 tracing_map_clear(hist_data->map);
2691
5463bfda
TZ		 2692 if (data->name)
2693 unpause_named_trigger(data);
2694}
2695
2696static bool compatible_field(struct ftrace_event_field *field,
2697 struct ftrace_event_field *test_field)
2698{
2699 if (field == test_field)
2700 return true;
2701 if (field == NULL || test_field == NULL)
2702 return false;
2703 if (strcmp(field->name, test_field->name) != 0)
2704 return false;
2705 if (strcmp(field->type, test_field->type) != 0)
2706 return false;
2707 if (field->size != test_field->size)
2708 return false;
2709 if (field->is_signed != test_field->is_signed)
2710 return false;
2711
2712 return true;
e86ae9ba
TZ		 2713}
2714
52a7f16d 2715static bool hist_trigger_match(struct event_trigger_data *data,
5463bfda
TZ		 2716 struct event_trigger_data *data_test,
2717 struct event_trigger_data *named_data,
2718 bool ignore_filter)
52a7f16d
TZ		 2719{
2720 struct tracing_map_sort_key *sort_key, *sort_key_test;
2721 struct hist_trigger_data *hist_data, *hist_data_test;
2722 struct hist_field *key_field, *key_field_test;
2723 unsigned int i;
2724
5463bfda
TZ		 2725 if (named_data && (named_data != data_test) &&
2726 (named_data != data_test->named_data))
2727 return false;
2728
2729 if (!named_data && is_named_trigger(data_test))
2730 return false;
2731
52a7f16d
TZ		 2732 hist_data = data->private_data;
2733 hist_data_test = data_test->private_data;
2734
2735 if (hist_data->n_vals != hist_data_test->n_vals ||
2736 hist_data->n_fields != hist_data_test->n_fields ||
2737 hist_data->n_sort_keys != hist_data_test->n_sort_keys)
2738 return false;
2739
5463bfda
TZ		 2740 if (!ignore_filter) {
2741 if ((data->filter_str && !data_test->filter_str) ||
2742 (!data->filter_str && data_test->filter_str))
2743 return false;
2744 }
52a7f16d
TZ		 2745
2746 for_each_hist_field(i, hist_data) {
2747 key_field = hist_data->fields[i];
2748 key_field_test = hist_data_test->fields[i];
2749
2750 if (key_field->flags != key_field_test->flags)
2751 return false;
5463bfda 2752 if (!compatible_field(key_field->field, key_field_test->field))
52a7f16d
TZ		 2753 return false;
2754 if (key_field->offset != key_field_test->offset)
2755 return false;
ad42febe
TZ		 2756 if (key_field->size != key_field_test->size)
2757 return false;
2758 if (key_field->is_signed != key_field_test->is_signed)
2759 return false;
1a361dfc
TZ		 2760 if (!!key_field->var.name != !!key_field_test->var.name)
2761 return false;
2762 if (key_field->var.name &&
2763 strcmp(key_field->var.name, key_field_test->var.name) != 0)
2764 return false;
52a7f16d
TZ		 2765 }
2766
2767 for (i = 0; i < hist_data->n_sort_keys; i++) {
2768 sort_key = &hist_data->sort_keys[i];
2769 sort_key_test = &hist_data_test->sort_keys[i];
2770
2771 if (sort_key->field_idx != sort_key_test->field_idx ||
2772 sort_key->descending != sort_key_test->descending)
2773 return false;
2774 }
2775
5463bfda 2776 if (!ignore_filter && data->filter_str &&
52a7f16d
TZ		 2777 (strcmp(data->filter_str, data_test->filter_str) != 0))
2778 return false;
2779
2780 return true;
2781}
2782
7ef224d1
TZ		 2783static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
2784 struct event_trigger_data *data,
2785 struct trace_event_file *file)
2786{
83e99914 2787 struct hist_trigger_data *hist_data = data->private_data;
5463bfda 2788 struct event_trigger_data *test, *named_data = NULL;
7ef224d1
TZ		 2789 int ret = 0;
2790
5463bfda
TZ		 2791 if (hist_data->attrs->name) {
2792 named_data = find_named_trigger(hist_data->attrs->name);
2793 if (named_data) {
2794 if (!hist_trigger_match(data, named_data, named_data,
2795 true)) {
2796 ret = -EINVAL;
2797 goto out;
2798 }
2799 }
2800 }
2801
2802 if (hist_data->attrs->name && !named_data)
2803 goto new;
2804
7ef224d1
TZ		 2805 list_for_each_entry_rcu(test, &file->triggers, list) {
2806 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 2807 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d 2808 continue;
83e99914
TZ		 2809 if (hist_data->attrs->pause)
2810 test->paused = true;
2811 else if (hist_data->attrs->cont)
2812 test->paused = false;
e86ae9ba
TZ		 2813 else if (hist_data->attrs->clear)
2814 hist_clear(test);
83e99914
TZ		 2815 else
2816 ret = -EEXIST;
7ef224d1
TZ		 2817 goto out;
2818 }
2819 }
5463bfda 2820 new:
e86ae9ba 2821 if (hist_data->attrs->cont || hist_data->attrs->clear) {
83e99914
TZ		 2822 ret = -ENOENT;
2823 goto out;
2824 }
2825
7522c03a
TZ		 2826 if (hist_data->attrs->pause)
2827 data->paused = true;
2828
5463bfda
TZ		 2829 if (named_data) {
2830 destroy_hist_data(data->private_data);
2831 data->private_data = named_data->private_data;
2832 set_named_trigger_data(data, named_data);
2833 data->ops = &event_hist_trigger_named_ops;
2834 }
2835
7ef224d1
TZ		 2836 if (data->ops->init) {
2837 ret = data->ops->init(data->ops, data);
2838 if (ret < 0)
2839 goto out;
2840 }
2841
7ef224d1
TZ		 2842 ret++;
2843
ad42febe
TZ		 2844 if (hist_data->enable_timestamps)
2845 tracing_set_time_stamp_abs(file->tr, true);
067fe038
TZ		 2846 out:
2847 return ret;
2848}
2849
2850static int hist_trigger_enable(struct event_trigger_data *data,
2851 struct trace_event_file *file)
2852{
2853 int ret = 0;
2854
2855 list_add_tail_rcu(&data->list, &file->triggers);
2856
2857 update_cond_flag(file);
ad42febe 2858
7ef224d1
TZ		 2859 if (trace_event_trigger_enable_disable(file, 1) < 0) {
2860 list_del_rcu(&data->list);
2861 update_cond_flag(file);
2862 ret--;
2863 }
067fe038 2864
7ef224d1
TZ		 2865 return ret;
2866}
2867
067fe038
TZ		 2868static bool hist_trigger_check_refs(struct event_trigger_data *data,
2869 struct trace_event_file *file)
2870{
2871 struct hist_trigger_data *hist_data = data->private_data;
2872 struct event_trigger_data *test, *named_data = NULL;
2873
2874 if (hist_data->attrs->name)
2875 named_data = find_named_trigger(hist_data->attrs->name);
2876
2877 list_for_each_entry_rcu(test, &file->triggers, list) {
2878 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2879 if (!hist_trigger_match(data, test, named_data, false))
2880 continue;
2881 hist_data = test->private_data;
2882 if (check_var_refs(hist_data))
2883 return true;
2884 break;
2885 }
2886 }
2887
2888 return false;
2889}
2890
52a7f16d
TZ		 2891static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
2892 struct event_trigger_data *data,
2893 struct trace_event_file *file)
2894{
5463bfda
TZ		 2895 struct hist_trigger_data *hist_data = data->private_data;
2896 struct event_trigger_data *test, *named_data = NULL;
52a7f16d
TZ		 2897 bool unregistered = false;
2898
5463bfda
TZ		 2899 if (hist_data->attrs->name)
2900 named_data = find_named_trigger(hist_data->attrs->name);
2901
52a7f16d
TZ		 2902 list_for_each_entry_rcu(test, &file->triggers, list) {
2903 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 2904 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d
TZ		 2905 continue;
2906 unregistered = true;
2907 list_del_rcu(&test->list);
2908 trace_event_trigger_enable_disable(file, 0);
2909 update_cond_flag(file);
2910 break;
2911 }
2912 }
2913
2914 if (unregistered && test->ops->free)
2915 test->ops->free(test->ops, test);
ad42febe
TZ		 2916
2917 if (hist_data->enable_timestamps) {
30350d65 2918 if (!hist_data->remove || unregistered)
ad42febe
TZ		 2919 tracing_set_time_stamp_abs(file->tr, false);
2920 }
52a7f16d
TZ		 2921}
2922
067fe038
TZ		 2923static bool hist_file_check_refs(struct trace_event_file *file)
2924{
2925 struct hist_trigger_data *hist_data;
2926 struct event_trigger_data *test;
2927
2928 list_for_each_entry_rcu(test, &file->triggers, list) {
2929 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2930 hist_data = test->private_data;
2931 if (check_var_refs(hist_data))
2932 return true;
2933 }
2934 }
2935
2936 return false;
2937}
2938
52a7f16d
TZ		 2939static void hist_unreg_all(struct trace_event_file *file)
2940{
47c18569 2941 struct event_trigger_data *test, *n;
ad42febe 2942 struct hist_trigger_data *hist_data;
52a7f16d 2943
067fe038
TZ		 2944 if (hist_file_check_refs(file))
2945 return;
2946
47c18569 2947 list_for_each_entry_safe(test, n, &file->triggers, list) {
52a7f16d 2948 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
ad42febe 2949 hist_data = test->private_data;
52a7f16d
TZ		 2950 list_del_rcu(&test->list);
2951 trace_event_trigger_enable_disable(file, 0);
2952 update_cond_flag(file);
ad42febe
TZ		 2953 if (hist_data->enable_timestamps)
2954 tracing_set_time_stamp_abs(file->tr, false);
52a7f16d
TZ		 2955 if (test->ops->free)
2956 test->ops->free(test->ops, test);
2957 }
2958 }
2959}
2960
7ef224d1
TZ		 2961static int event_hist_trigger_func(struct event_command *cmd_ops,
2962 struct trace_event_file *file,
2963 char *glob, char *cmd, char *param)
2964{
2965 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
2966 struct event_trigger_data *trigger_data;
2967 struct hist_trigger_attrs *attrs;
2968 struct event_trigger_ops *trigger_ops;
2969 struct hist_trigger_data *hist_data;
30350d65 2970 bool remove = false;
7ef224d1
TZ		 2971 char *trigger;
2972 int ret = 0;
2973
2974 if (!param)
2975 return -EINVAL;
2976
30350d65
TZ		 2977 if (glob[0] == '!')
2978 remove = true;
2979
7ef224d1
TZ		 2980 /* separate the trigger from the filter (k:v [if filter]) */
2981 trigger = strsep(&param, " \t");
2982 if (!trigger)
2983 return -EINVAL;
2984
2985 attrs = parse_hist_trigger_attrs(trigger);
2986 if (IS_ERR(attrs))
2987 return PTR_ERR(attrs);
2988
2989 if (attrs->map_bits)
2990 hist_trigger_bits = attrs->map_bits;
2991
30350d65 2992 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
7ef224d1
TZ		 2993 if (IS_ERR(hist_data)) {
2994 destroy_hist_trigger_attrs(attrs);
2995 return PTR_ERR(hist_data);
2996 }
2997
2998 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
2999
3000 ret = -ENOMEM;
3001 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
3002 if (!trigger_data)
3003 goto out_free;
3004
3005 trigger_data->count = -1;
3006 trigger_data->ops = trigger_ops;
3007 trigger_data->cmd_ops = cmd_ops;
3008
3009 INIT_LIST_HEAD(&trigger_data->list);
3010 RCU_INIT_POINTER(trigger_data->filter, NULL);
3011
3012 trigger_data->private_data = hist_data;
3013
52a7f16d
TZ		 3014 /* if param is non-empty, it's supposed to be a filter */
3015 if (param && cmd_ops->set_filter) {
3016 ret = cmd_ops->set_filter(param, trigger_data, file);
3017 if (ret < 0)
3018 goto out_free;
3019 }
3020
30350d65 3021 if (remove) {
067fe038
TZ		 3022 if (hist_trigger_check_refs(trigger_data, file)) {
3023 ret = -EBUSY;
3024 goto out_free;
3025 }
3026
7ef224d1
TZ		 3027 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
3028 ret = 0;
3029 goto out_free;
3030 }
3031
7ef224d1
TZ		 3032 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
3033 /*
3034 * The above returns on success the # of triggers registered,
3035 * but if it didn't register any it returns zero. Consider no
3036 * triggers registered a failure too.
3037 */
3038 if (!ret) {
e86ae9ba 3039 if (!(attrs->pause || attrs->cont || attrs->clear))
83e99914 3040 ret = -ENOENT;
7ef224d1
TZ		 3041 goto out_free;
3042 } else if (ret < 0)
3043 goto out_free;
067fe038
TZ		 3044
3045 if (get_named_trigger_data(trigger_data))
3046 goto enable;
3047
3048 if (has_hist_vars(hist_data))
3049 save_hist_vars(hist_data);
3050
3051 ret = tracing_map_init(hist_data->map);
3052 if (ret)
3053 goto out_unreg;
3054enable:
3055 ret = hist_trigger_enable(trigger_data, file);
3056 if (ret)
3057 goto out_unreg;
3058
7ef224d1
TZ		 3059 /* Just return zero, not the number of registered triggers */
3060 ret = 0;
3061 out:
3062 return ret;
067fe038
TZ		 3063 out_unreg:
3064 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
7ef224d1
TZ		 3065 out_free:
3066 if (cmd_ops->set_filter)
3067 cmd_ops->set_filter(NULL, trigger_data, NULL);
3068
067fe038
TZ		 3069 remove_hist_vars(hist_data);
3070
7ef224d1
TZ		 3071 kfree(trigger_data);
3072
3073 destroy_hist_data(hist_data);
3074 goto out;
3075}
3076
3077static struct event_command trigger_hist_cmd = {
3078 .name = "hist",
3079 .trigger_type = ETT_EVENT_HIST,
3080 .flags = EVENT_CMD_FL_NEEDS_REC,
3081 .func = event_hist_trigger_func,
3082 .reg = hist_register_trigger,
52a7f16d
TZ		 3083 .unreg = hist_unregister_trigger,
3084 .unreg_all = hist_unreg_all,
7ef224d1
TZ		 3085 .get_trigger_ops = event_hist_get_trigger_ops,
3086 .set_filter = set_trigger_filter,
3087};
3088
3089__init int register_trigger_hist_cmd(void)
3090{
3091 int ret;
3092
3093 ret = register_event_command(&trigger_hist_cmd);
3094 WARN_ON(ret < 0);
3095
3096 return ret;
3097}
d0bad49b
TZ		 3098
3099static void
1ac4f51c
TZ		 3100hist_enable_trigger(struct event_trigger_data *data, void *rec,
3101 struct ring_buffer_event *event)
d0bad49b
TZ		 3102{
3103 struct enable_trigger_data *enable_data = data->private_data;
3104 struct event_trigger_data *test;
3105
3106 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
3107 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3108 if (enable_data->enable)
3109 test->paused = false;
3110 else
3111 test->paused = true;
d0bad49b
TZ		 3112 }
3113 }
3114}
3115
3116static void
1ac4f51c
TZ		 3117hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
3118 struct ring_buffer_event *event)
d0bad49b
TZ		 3119{
3120 if (!data->count)
3121 return;
3122
3123 if (data->count != -1)
3124 (data->count)--;
3125
1ac4f51c 3126 hist_enable_trigger(data, rec, event);
d0bad49b
TZ		 3127}
3128
3129static struct event_trigger_ops hist_enable_trigger_ops = {
3130 .func = hist_enable_trigger,
3131 .print = event_enable_trigger_print,
3132 .init = event_trigger_init,
3133 .free = event_enable_trigger_free,
3134};
3135
3136static struct event_trigger_ops hist_enable_count_trigger_ops = {
3137 .func = hist_enable_count_trigger,
3138 .print = event_enable_trigger_print,
3139 .init = event_trigger_init,
3140 .free = event_enable_trigger_free,
3141};
3142
3143static struct event_trigger_ops hist_disable_trigger_ops = {
3144 .func = hist_enable_trigger,
3145 .print = event_enable_trigger_print,
3146 .init = event_trigger_init,
3147 .free = event_enable_trigger_free,
3148};
3149
3150static struct event_trigger_ops hist_disable_count_trigger_ops = {
3151 .func = hist_enable_count_trigger,
3152 .print = event_enable_trigger_print,
3153 .init = event_trigger_init,
3154 .free = event_enable_trigger_free,
3155};
3156
3157static struct event_trigger_ops *
3158hist_enable_get_trigger_ops(char *cmd, char *param)
3159{
3160 struct event_trigger_ops *ops;
3161 bool enable;
3162
3163 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
3164
3165 if (enable)
3166 ops = param ? &hist_enable_count_trigger_ops :
3167 &hist_enable_trigger_ops;
3168 else
3169 ops = param ? &hist_disable_count_trigger_ops :
3170 &hist_disable_trigger_ops;
3171
3172 return ops;
3173}
3174
52a7f16d
TZ		 3175static void hist_enable_unreg_all(struct trace_event_file *file)
3176{
47c18569 3177 struct event_trigger_data *test, *n;
52a7f16d 3178
47c18569 3179 list_for_each_entry_safe(test, n, &file->triggers, list) {
52a7f16d
TZ		 3180 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
3181 list_del_rcu(&test->list);
3182 update_cond_flag(file);
3183 trace_event_trigger_enable_disable(file, 0);
3184 if (test->ops->free)
3185 test->ops->free(test->ops, test);
3186 }
3187 }
3188}
3189
d0bad49b
TZ		 3190static struct event_command trigger_hist_enable_cmd = {
3191 .name = ENABLE_HIST_STR,
3192 .trigger_type = ETT_HIST_ENABLE,
3193 .func = event_enable_trigger_func,
3194 .reg = event_enable_register_trigger,
3195 .unreg = event_enable_unregister_trigger,
52a7f16d 3196 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 3197 .get_trigger_ops = hist_enable_get_trigger_ops,
3198 .set_filter = set_trigger_filter,
3199};
3200
3201static struct event_command trigger_hist_disable_cmd = {
3202 .name = DISABLE_HIST_STR,
3203 .trigger_type = ETT_HIST_ENABLE,
3204 .func = event_enable_trigger_func,
3205 .reg = event_enable_register_trigger,
3206 .unreg = event_enable_unregister_trigger,
52a7f16d 3207 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 3208 .get_trigger_ops = hist_enable_get_trigger_ops,
3209 .set_filter = set_trigger_filter,
3210};
3211
3212static __init void unregister_trigger_hist_enable_disable_cmds(void)
3213{
3214 unregister_event_command(&trigger_hist_enable_cmd);
3215 unregister_event_command(&trigger_hist_disable_cmd);
3216}
3217
3218__init int register_trigger_hist_enable_disable_cmds(void)
3219{
3220 int ret;
3221
3222 ret = register_event_command(&trigger_hist_enable_cmd);
3223 if (WARN_ON(ret < 0))
3224 return ret;
3225 ret = register_event_command(&trigger_hist_disable_cmd);
3226 if (WARN_ON(ret < 0))
3227 unregister_trigger_hist_enable_disable_cmds();
3228
3229 return ret;
3230}
Linux 6.x block layer and io_uring tree(s)