kernel/trace/trace_probe.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common code for probe-based Dynamic events.
   4  *
   5  * This code was copied from kernel/trace/trace_kprobe.c written by
   6  * Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
   7  *
   8  * Updates to make this generic:
   9  * Copyright (C) IBM Corporation, 2010-2011
  10  * Author:     Srikar Dronamraju
  11  */
  12 #define pr_fmt(fmt)     "trace_probe: " fmt
  13
  14 #include <linux/bpf.h>
  15 #include <linux/fs.h>
  16 #include "trace_btf.h"
  17
  18 #include "trace_probe.h"
  19
  20 #undef C
  21 #define C(a, b)         b
  22
  23 static const char *trace_probe_err_text[] = { ERRORS };
  24
  25 static const char *reserved_field_names[] = {
  26         "common_type",
  27         "common_flags",
  28         "common_preempt_count",
  29         "common_pid",
  30         "common_tgid",
  31         FIELD_STRING_IP,
  32         FIELD_STRING_RETIP,
  33         FIELD_STRING_FUNC,
  34 };
  35
  36 /* Printing  in basic type function template */
  37 #define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt)                  \
  38 int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
  39 {                                                                       \
  40         trace_seq_printf(s, fmt, *(type *)data);                        \
  41         return !trace_seq_has_overflowed(s);                            \
  42 }                                                                       \
  43 const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
  44
  45 DEFINE_BASIC_PRINT_TYPE_FUNC(u8,  u8,  "%u")
  46 DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
  47 DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u")
  48 DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu")
  49 DEFINE_BASIC_PRINT_TYPE_FUNC(s8,  s8,  "%d")
  50 DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d")
  51 DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d")
  52 DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld")
  53 DEFINE_BASIC_PRINT_TYPE_FUNC(x8,  u8,  "0x%x")
  54 DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
  55 DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
  56 DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
  57 DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'")
  58
  59 int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
  60 {
  61         trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data);
  62         return !trace_seq_has_overflowed(s);
  63 }
  64 const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
  65
  66 /* Print type function for string type */
  67 int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
  68 {
  69         int len = *(u32 *)data >> 16;
  70
  71         if (!len)
  72                 trace_seq_puts(s, FAULT_STRING);
  73         else
  74                 trace_seq_printf(s, "\"%s\"",
  75                                  (const char *)get_loc_data(data, ent));
  76         return !trace_seq_has_overflowed(s);
  77 }
  78
  79 const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
  80
  81 /* Fetch type information table */
  82 static const struct fetch_type probe_fetch_types[] = {
  83         /* Special types */
  84         __ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, 1,
  85                             "__data_loc char[]"),
  86         __ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, 1,
  87                             "__data_loc char[]"),
  88         __ASSIGN_FETCH_TYPE("symstr", string, string, sizeof(u32), 1, 1,
  89                             "__data_loc char[]"),
  90         /* Basic types */
  91         ASSIGN_FETCH_TYPE(u8,  u8,  0),
  92         ASSIGN_FETCH_TYPE(u16, u16, 0),
  93         ASSIGN_FETCH_TYPE(u32, u32, 0),
  94         ASSIGN_FETCH_TYPE(u64, u64, 0),
  95         ASSIGN_FETCH_TYPE(s8,  u8,  1),
  96         ASSIGN_FETCH_TYPE(s16, u16, 1),
  97         ASSIGN_FETCH_TYPE(s32, u32, 1),
  98         ASSIGN_FETCH_TYPE(s64, u64, 1),
  99         ASSIGN_FETCH_TYPE_ALIAS(x8,  u8,  u8,  0),
 100         ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
 101         ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
 102         ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
 103         ASSIGN_FETCH_TYPE_ALIAS(char, u8, u8,  0),
 104         ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
 105
 106         ASSIGN_FETCH_TYPE_END
 107 };
 108
 109 static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags)
 110 {
 111         int i;
 112
 113         /* Reject the symbol/symstr for uprobes */
 114         if (type && (flags & TPARG_FL_USER) &&
 115             (!strcmp(type, "symbol") || !strcmp(type, "symstr")))
 116                 return NULL;
 117
 118         if (!type)
 119                 type = DEFAULT_FETCH_TYPE_STR;
 120
 121         /* Special case: bitfield */
 122         if (*type == 'b') {
 123                 unsigned long bs;
 124
 125                 type = strchr(type, '/');
 126                 if (!type)
 127                         goto fail;
 128
 129                 type++;
 130                 if (kstrtoul(type, 0, &bs))
 131                         goto fail;
 132
 133                 switch (bs) {
 134                 case 8:
 135                         return find_fetch_type("u8", flags);
 136                 case 16:
 137                         return find_fetch_type("u16", flags);
 138                 case 32:
 139                         return find_fetch_type("u32", flags);
 140                 case 64:
 141                         return find_fetch_type("u64", flags);
 142                 default:
 143                         goto fail;
 144                 }
 145         }
 146
 147         for (i = 0; probe_fetch_types[i].name; i++) {
 148                 if (strcmp(type, probe_fetch_types[i].name) == 0)
 149                         return &probe_fetch_types[i];
 150         }
 151
 152 fail:
 153         return NULL;
 154 }
 155
 156 static struct trace_probe_log trace_probe_log;
 157 extern struct mutex dyn_event_ops_mutex;
 158
 159 void trace_probe_log_init(const char *subsystem, int argc, const char **argv)
 160 {
 161         lockdep_assert_held(&dyn_event_ops_mutex);
 162
 163         trace_probe_log.subsystem = subsystem;
 164         trace_probe_log.argc = argc;
 165         trace_probe_log.argv = argv;
 166         trace_probe_log.index = 0;
 167 }
 168
 169 void trace_probe_log_clear(void)
 170 {
 171         lockdep_assert_held(&dyn_event_ops_mutex);
 172
 173         memset(&trace_probe_log, 0, sizeof(trace_probe_log));
 174 }
 175
 176 void trace_probe_log_set_index(int index)
 177 {
 178         lockdep_assert_held(&dyn_event_ops_mutex);
 179
 180         trace_probe_log.index = index;
 181 }
 182
 183 void __trace_probe_log_err(int offset, int err_type)
 184 {
 185         char *command, *p;
 186         int i, len = 0, pos = 0;
 187
 188         lockdep_assert_held(&dyn_event_ops_mutex);
 189
 190         if (!trace_probe_log.argv)
 191                 return;
 192
 193         /* Recalculate the length and allocate buffer */
 194         for (i = 0; i < trace_probe_log.argc; i++) {
 195                 if (i == trace_probe_log.index)
 196                         pos = len;
 197                 len += strlen(trace_probe_log.argv[i]) + 1;
 198         }
 199         command = kzalloc(len, GFP_KERNEL);
 200         if (!command)
 201                 return;
 202
 203         if (trace_probe_log.index >= trace_probe_log.argc) {
 204                 /**
 205                  * Set the error position is next to the last arg + space.
 206                  * Note that len includes the terminal null and the cursor
 207                  * appears at pos + 1.
 208                  */
 209                 pos = len;
 210                 offset = 0;
 211         }
 212
 213         /* And make a command string from argv array */
 214         p = command;
 215         for (i = 0; i < trace_probe_log.argc; i++) {
 216                 len = strlen(trace_probe_log.argv[i]);
 217                 strcpy(p, trace_probe_log.argv[i]);
 218                 p[len] = ' ';
 219                 p += len + 1;
 220         }
 221         *(p - 1) = '\0';
 222
 223         tracing_log_err(NULL, trace_probe_log.subsystem, command,
 224                         trace_probe_err_text, err_type, pos + offset);
 225
 226         kfree(command);
 227 }
 228
 229 /* Split symbol and offset. */
 230 int traceprobe_split_symbol_offset(char *symbol, long *offset)
 231 {
 232         char *tmp;
 233         int ret;
 234
 235         if (!offset)
 236                 return -EINVAL;
 237
 238         tmp = strpbrk(symbol, "+-");
 239         if (tmp) {
 240                 ret = kstrtol(tmp, 0, offset);
 241                 if (ret)
 242                         return ret;
 243                 *tmp = '\0';
 244         } else
 245                 *offset = 0;
 246
 247         return 0;
 248 }
 249
 250 /* @buf must has MAX_EVENT_NAME_LEN size */
 251 int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
 252                                 char *buf, int offset)
 253 {
 254         const char *slash, *event = *pevent;
 255         int len;
 256
 257         slash = strchr(event, '/');
 258         if (!slash)
 259                 slash = strchr(event, '.');
 260
 261         if (slash) {
 262                 if (slash == event) {
 263                         trace_probe_log_err(offset, NO_GROUP_NAME);
 264                         return -EINVAL;
 265                 }
 266                 if (slash - event + 1 > MAX_EVENT_NAME_LEN) {
 267                         trace_probe_log_err(offset, GROUP_TOO_LONG);
 268                         return -EINVAL;
 269                 }
 270                 strscpy(buf, event, slash - event + 1);
 271                 if (!is_good_system_name(buf)) {
 272                         trace_probe_log_err(offset, BAD_GROUP_NAME);
 273                         return -EINVAL;
 274                 }
 275                 *pgroup = buf;
 276                 *pevent = slash + 1;
 277                 offset += slash - event + 1;
 278                 event = *pevent;
 279         }
 280         len = strlen(event);
 281         if (len == 0) {
 282                 if (slash) {
 283                         *pevent = NULL;
 284                         return 0;
 285                 }
 286                 trace_probe_log_err(offset, NO_EVENT_NAME);
 287                 return -EINVAL;
 288         } else if (len >= MAX_EVENT_NAME_LEN) {
 289                 trace_probe_log_err(offset, EVENT_TOO_LONG);
 290                 return -EINVAL;
 291         }
 292         if (!is_good_name(event)) {
 293                 trace_probe_log_err(offset, BAD_EVENT_NAME);
 294                 return -EINVAL;
 295         }
 296         return 0;
 297 }
 298
 299 static int parse_trace_event_arg(char *arg, struct fetch_insn *code,
 300                                  struct traceprobe_parse_context *ctx)
 301 {
 302         struct ftrace_event_field *field;
 303         struct list_head *head;
 304
 305         head = trace_get_fields(ctx->event);
 306         list_for_each_entry(field, head, link) {
 307                 if (!strcmp(arg, field->name)) {
 308                         code->op = FETCH_OP_TP_ARG;
 309                         code->data = field;
 310                         return 0;
 311                 }
 312         }
 313         return -ENOENT;
 314 }
 315
 316 #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
 317
 318 static u32 btf_type_int(const struct btf_type *t)
 319 {
 320         return *(u32 *)(t + 1);
 321 }
 322
 323 static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type)
 324 {
 325         const struct btf_type *real_type;
 326         u32 intdata;
 327         s32 tid;
 328
 329         real_type = btf_type_skip_modifiers(btf, type->type, &tid);
 330         if (!real_type)
 331                 return false;
 332
 333         if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT)
 334                 return false;
 335
 336         intdata = btf_type_int(real_type);
 337         return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
 338                 && BTF_INT_BITS(intdata) == 8;
 339 }
 340
 341 static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type)
 342 {
 343         const struct btf_type *real_type;
 344         const struct btf_array *array;
 345         u32 intdata;
 346         s32 tid;
 347
 348         if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY)
 349                 return false;
 350
 351         array = (const struct btf_array *)(type + 1);
 352
 353         real_type = btf_type_skip_modifiers(btf, array->type, &tid);
 354
 355         intdata = btf_type_int(real_type);
 356         return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
 357                 && BTF_INT_BITS(intdata) == 8;
 358 }
 359
 360 static int check_prepare_btf_string_fetch(char *typename,
 361                                 struct fetch_insn **pcode,
 362                                 struct traceprobe_parse_context *ctx)
 363 {
 364         struct btf *btf = ctx->btf;
 365
 366         if (!btf || !ctx->last_type)
 367                 return 0;
 368
 369         /* char [] does not need any change. */
 370         if (btf_type_is_char_array(btf, ctx->last_type))
 371                 return 0;
 372
 373         /* char * requires dereference the pointer. */
 374         if (btf_type_is_char_ptr(btf, ctx->last_type)) {
 375                 struct fetch_insn *code = *pcode + 1;
 376
 377                 if (code->op == FETCH_OP_END) {
 378                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 379                         return -E2BIG;
 380                 }
 381                 if (typename[0] == 'u')
 382                         code->op = FETCH_OP_UDEREF;
 383                 else
 384                         code->op = FETCH_OP_DEREF;
 385                 code->offset = 0;
 386                 *pcode = code;
 387                 return 0;
 388         }
 389         /* Other types are not available for string */
 390         trace_probe_log_err(ctx->offset, BAD_TYPE4STR);
 391         return -EINVAL;
 392 }
 393
 394 static const char *fetch_type_from_btf_type(struct btf *btf,
 395                                         const struct btf_type *type,
 396                                         struct traceprobe_parse_context *ctx)
 397 {
 398         u32 intdata;
 399
 400         /* TODO: const char * could be converted as a string */
 401         switch (BTF_INFO_KIND(type->info)) {
 402         case BTF_KIND_ENUM:
 403                 /* enum is "int", so convert to "s32" */
 404                 return "s32";
 405         case BTF_KIND_ENUM64:
 406                 return "s64";
 407         case BTF_KIND_PTR:
 408                 /* pointer will be converted to "x??" */
 409                 if (IS_ENABLED(CONFIG_64BIT))
 410                         return "x64";
 411                 else
 412                         return "x32";
 413         case BTF_KIND_INT:
 414                 intdata = btf_type_int(type);
 415                 if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) {
 416                         switch (BTF_INT_BITS(intdata)) {
 417                         case 8:
 418                                 return "s8";
 419                         case 16:
 420                                 return "s16";
 421                         case 32:
 422                                 return "s32";
 423                         case 64:
 424                                 return "s64";
 425                         }
 426                 } else {        /* unsigned */
 427                         switch (BTF_INT_BITS(intdata)) {
 428                         case 8:
 429                                 return "u8";
 430                         case 16:
 431                                 return "u16";
 432                         case 32:
 433                                 return "u32";
 434                         case 64:
 435                                 return "u64";
 436                         }
 437                         /* bitfield, size is encoded in the type */
 438                         ctx->last_bitsize = BTF_INT_BITS(intdata);
 439                         ctx->last_bitoffs += BTF_INT_OFFSET(intdata);
 440                         return "u64";
 441                 }
 442         }
 443         /* TODO: support other types */
 444
 445         return NULL;
 446 }
 447
 448 static int query_btf_context(struct traceprobe_parse_context *ctx)
 449 {
 450         const struct btf_param *param;
 451         const struct btf_type *type;
 452         struct btf *btf;
 453         s32 nr;
 454
 455         if (ctx->btf)
 456                 return 0;
 457
 458         if (!ctx->funcname)
 459                 return -EINVAL;
 460
 461         type = btf_find_func_proto(ctx->funcname, &btf);
 462         if (!type)
 463                 return -ENOENT;
 464
 465         ctx->btf = btf;
 466         ctx->proto = type;
 467
 468         /* ctx->params is optional, since func(void) will not have params. */
 469         nr = 0;
 470         param = btf_get_func_param(type, &nr);
 471         if (!IS_ERR_OR_NULL(param)) {
 472                 /* Hide the first 'data' argument of tracepoint */
 473                 if (ctx->flags & TPARG_FL_TPOINT) {
 474                         nr--;
 475                         param++;
 476                 }
 477         }
 478
 479         if (nr > 0) {
 480                 ctx->nr_params = nr;
 481                 ctx->params = param;
 482         } else {
 483                 ctx->nr_params = 0;
 484                 ctx->params = NULL;
 485         }
 486
 487         return 0;
 488 }
 489
 490 static void clear_btf_context(struct traceprobe_parse_context *ctx)
 491 {
 492         if (ctx->btf) {
 493                 btf_put(ctx->btf);
 494                 ctx->btf = NULL;
 495                 ctx->proto = NULL;
 496                 ctx->params = NULL;
 497                 ctx->nr_params = 0;
 498         }
 499 }
 500
 501 /* Return 1 if the field separater is arrow operator ('->') */
 502 static int split_next_field(char *varname, char **next_field,
 503                             struct traceprobe_parse_context *ctx)
 504 {
 505         char *field;
 506         int ret = 0;
 507
 508         field = strpbrk(varname, ".-");
 509         if (field) {
 510                 if (field[0] == '-' && field[1] == '>') {
 511                         field[0] = '\0';
 512                         field += 2;
 513                         ret = 1;
 514                 } else if (field[0] == '.') {
 515                         field[0] = '\0';
 516                         field += 1;
 517                 } else {
 518                         trace_probe_log_err(ctx->offset + field - varname, BAD_HYPHEN);
 519                         return -EINVAL;
 520                 }
 521                 *next_field = field;
 522         }
 523
 524         return ret;
 525 }
 526
 527 /*
 528  * Parse the field of data structure. The @type must be a pointer type
 529  * pointing the target data structure type.
 530  */
 531 static int parse_btf_field(char *fieldname, const struct btf_type *type,
 532                            struct fetch_insn **pcode, struct fetch_insn *end,
 533                            struct traceprobe_parse_context *ctx)
 534 {
 535         struct fetch_insn *code = *pcode;
 536         const struct btf_member *field;
 537         u32 bitoffs, anon_offs;
 538         char *next;
 539         int is_ptr;
 540         s32 tid;
 541
 542         do {
 543                 /* Outer loop for solving arrow operator ('->') */
 544                 if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
 545                         trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
 546                         return -EINVAL;
 547                 }
 548                 /* Convert a struct pointer type to a struct type */
 549                 type = btf_type_skip_modifiers(ctx->btf, type->type, &tid);
 550                 if (!type) {
 551                         trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 552                         return -EINVAL;
 553                 }
 554
 555                 bitoffs = 0;
 556                 do {
 557                         /* Inner loop for solving dot operator ('.') */
 558                         next = NULL;
 559                         is_ptr = split_next_field(fieldname, &next, ctx);
 560                         if (is_ptr < 0)
 561                                 return is_ptr;
 562
 563                         anon_offs = 0;
 564                         field = btf_find_struct_member(ctx->btf, type, fieldname,
 565                                                        &anon_offs);
 566                         if (IS_ERR(field)) {
 567                                 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 568                                 return PTR_ERR(field);
 569                         }
 570                         if (!field) {
 571                                 trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
 572                                 return -ENOENT;
 573                         }
 574                         /* Add anonymous structure/union offset */
 575                         bitoffs += anon_offs;
 576
 577                         /* Accumulate the bit-offsets of the dot-connected fields */
 578                         if (btf_type_kflag(type)) {
 579                                 bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
 580                                 ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
 581                         } else {
 582                                 bitoffs += field->offset;
 583                                 ctx->last_bitsize = 0;
 584                         }
 585
 586                         type = btf_type_skip_modifiers(ctx->btf, field->type, &tid);
 587                         if (!type) {
 588                                 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 589                                 return -EINVAL;
 590                         }
 591
 592                         ctx->offset += next - fieldname;
 593                         fieldname = next;
 594                 } while (!is_ptr && fieldname);
 595
 596                 if (++code == end) {
 597                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 598                         return -EINVAL;
 599                 }
 600                 code->op = FETCH_OP_DEREF;      /* TODO: user deref support */
 601                 code->offset = bitoffs / 8;
 602                 *pcode = code;
 603
 604                 ctx->last_bitoffs = bitoffs % 8;
 605                 ctx->last_type = type;
 606         } while (fieldname);
 607
 608         return 0;
 609 }
 610
 611 static int __store_entry_arg(struct trace_probe *tp, int argnum);
 612
 613 static int parse_btf_arg(char *varname,
 614                          struct fetch_insn **pcode, struct fetch_insn *end,
 615                          struct traceprobe_parse_context *ctx)
 616 {
 617         struct fetch_insn *code = *pcode;
 618         const struct btf_param *params;
 619         const struct btf_type *type;
 620         char *field = NULL;
 621         int i, is_ptr, ret;
 622         u32 tid;
 623
 624         if (WARN_ON_ONCE(!ctx->funcname))
 625                 return -EINVAL;
 626
 627         is_ptr = split_next_field(varname, &field, ctx);
 628         if (is_ptr < 0)
 629                 return is_ptr;
 630         if (!is_ptr && field) {
 631                 /* dot-connected field on an argument is not supported. */
 632                 trace_probe_log_err(ctx->offset + field - varname,
 633                                     NOSUP_DAT_ARG);
 634                 return -EOPNOTSUPP;
 635         }
 636
 637         if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) {
 638                 code->op = FETCH_OP_RETVAL;
 639                 /* Check whether the function return type is not void */
 640                 if (query_btf_context(ctx) == 0) {
 641                         if (ctx->proto->type == 0) {
 642                                 trace_probe_log_err(ctx->offset, NO_RETVAL);
 643                                 return -ENOENT;
 644                         }
 645                         tid = ctx->proto->type;
 646                         goto found;
 647                 }
 648                 if (field) {
 649                         trace_probe_log_err(ctx->offset + field - varname,
 650                                             NO_BTF_ENTRY);
 651                         return -ENOENT;
 652                 }
 653                 return 0;
 654         }
 655
 656         if (!ctx->btf) {
 657                 ret = query_btf_context(ctx);
 658                 if (ret < 0 || ctx->nr_params == 0) {
 659                         trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
 660                         return PTR_ERR(params);
 661                 }
 662         }
 663         params = ctx->params;
 664
 665         for (i = 0; i < ctx->nr_params; i++) {
 666                 const char *name = btf_name_by_offset(ctx->btf, params[i].name_off);
 667
 668                 if (name && !strcmp(name, varname)) {
 669                         if (tparg_is_function_entry(ctx->flags)) {
 670                                 code->op = FETCH_OP_ARG;
 671                                 if (ctx->flags & TPARG_FL_TPOINT)
 672                                         code->param = i + 1;
 673                                 else
 674                                         code->param = i;
 675                         } else if (tparg_is_function_return(ctx->flags)) {
 676                                 code->op = FETCH_OP_EDATA;
 677                                 ret = __store_entry_arg(ctx->tp, i);
 678                                 if (ret < 0) {
 679                                         /* internal error */
 680                                         return ret;
 681                                 }
 682                                 code->offset = ret;
 683                         }
 684                         tid = params[i].type;
 685                         goto found;
 686                 }
 687         }
 688         trace_probe_log_err(ctx->offset, NO_BTFARG);
 689         return -ENOENT;
 690
 691 found:
 692         type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
 693         if (!type) {
 694                 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 695                 return -EINVAL;
 696         }
 697         /* Initialize the last type information */
 698         ctx->last_type = type;
 699         ctx->last_bitoffs = 0;
 700         ctx->last_bitsize = 0;
 701         if (field) {
 702                 ctx->offset += field - varname;
 703                 return parse_btf_field(field, type, pcode, end, ctx);
 704         }
 705         return 0;
 706 }
 707
 708 static const struct fetch_type *find_fetch_type_from_btf_type(
 709                                         struct traceprobe_parse_context *ctx)
 710 {
 711         struct btf *btf = ctx->btf;
 712         const char *typestr = NULL;
 713
 714         if (btf && ctx->last_type)
 715                 typestr = fetch_type_from_btf_type(btf, ctx->last_type, ctx);
 716
 717         return find_fetch_type(typestr, ctx->flags);
 718 }
 719
 720 static int parse_btf_bitfield(struct fetch_insn **pcode,
 721                               struct traceprobe_parse_context *ctx)
 722 {
 723         struct fetch_insn *code = *pcode;
 724
 725         if ((ctx->last_bitsize % 8 == 0) && ctx->last_bitoffs == 0)
 726                 return 0;
 727
 728         code++;
 729         if (code->op != FETCH_OP_NOP) {
 730                 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 731                 return -EINVAL;
 732         }
 733         *pcode = code;
 734
 735         code->op = FETCH_OP_MOD_BF;
 736         code->lshift = 64 - (ctx->last_bitsize + ctx->last_bitoffs);
 737         code->rshift = 64 - ctx->last_bitsize;
 738         code->basesize = 64 / 8;
 739         return 0;
 740 }
 741
 742 #else
 743 static void clear_btf_context(struct traceprobe_parse_context *ctx)
 744 {
 745         ctx->btf = NULL;
 746 }
 747
 748 static int query_btf_context(struct traceprobe_parse_context *ctx)
 749 {
 750         return -EOPNOTSUPP;
 751 }
 752
 753 static int parse_btf_arg(char *varname,
 754                          struct fetch_insn **pcode, struct fetch_insn *end,
 755                          struct traceprobe_parse_context *ctx)
 756 {
 757         trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
 758         return -EOPNOTSUPP;
 759 }
 760
 761 static int parse_btf_bitfield(struct fetch_insn **pcode,
 762                               struct traceprobe_parse_context *ctx)
 763 {
 764         trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
 765         return -EOPNOTSUPP;
 766 }
 767
 768 #define find_fetch_type_from_btf_type(ctx)              \
 769         find_fetch_type(NULL, ctx->flags)
 770
 771 static int check_prepare_btf_string_fetch(char *typename,
 772                                 struct fetch_insn **pcode,
 773                                 struct traceprobe_parse_context *ctx)
 774 {
 775         return 0;
 776 }
 777
 778 #endif
 779
 780 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
 781
 782 /*
 783  * Add the entry code to store the 'argnum'th parameter and return the offset
 784  * in the entry data buffer where the data will be stored.
 785  */
 786 static int __store_entry_arg(struct trace_probe *tp, int argnum)
 787 {
 788         struct probe_entry_arg *earg = tp->entry_arg;
 789         bool match = false;
 790         int i, offset;
 791
 792         if (!earg) {
 793                 earg = kzalloc(sizeof(*tp->entry_arg), GFP_KERNEL);
 794                 if (!earg)
 795                         return -ENOMEM;
 796                 earg->size = 2 * tp->nr_args + 1;
 797                 earg->code = kcalloc(earg->size, sizeof(struct fetch_insn),
 798                                      GFP_KERNEL);
 799                 if (!earg->code) {
 800                         kfree(earg);
 801                         return -ENOMEM;
 802                 }
 803                 /* Fill the code buffer with 'end' to simplify it */
 804                 for (i = 0; i < earg->size; i++)
 805                         earg->code[i].op = FETCH_OP_END;
 806                 tp->entry_arg = earg;
 807         }
 808
 809         /*
 810          * The entry code array is repeating the pair of
 811          * [FETCH_OP_ARG(argnum)][FETCH_OP_ST_EDATA(offset of entry data buffer)]
 812          * and the rest of entries are filled with [FETCH_OP_END].
 813          *
 814          * To reduce the redundant function parameter fetching, we scan the entry
 815          * code array to find the FETCH_OP_ARG which already fetches the 'argnum'
 816          * parameter. If it doesn't match, update 'offset' to find the last
 817          * offset.
 818          * If we find the FETCH_OP_END without matching FETCH_OP_ARG entry, we
 819          * will save the entry with FETCH_OP_ARG and FETCH_OP_ST_EDATA, and
 820          * return data offset so that caller can find the data offset in the entry
 821          * data buffer.
 822          */
 823         offset = 0;
 824         for (i = 0; i < earg->size - 1; i++) {
 825                 switch (earg->code[i].op) {
 826                 case FETCH_OP_END:
 827                         earg->code[i].op = FETCH_OP_ARG;
 828                         earg->code[i].param = argnum;
 829                         earg->code[i + 1].op = FETCH_OP_ST_EDATA;
 830                         earg->code[i + 1].offset = offset;
 831                         return offset;
 832                 case FETCH_OP_ARG:
 833                         match = (earg->code[i].param == argnum);
 834                         break;
 835                 case FETCH_OP_ST_EDATA:
 836                         offset = earg->code[i].offset;
 837                         if (match)
 838                                 return offset;
 839                         offset += sizeof(unsigned long);
 840                         break;
 841                 default:
 842                         break;
 843                 }
 844         }
 845         return -ENOSPC;
 846 }
 847
 848 int traceprobe_get_entry_data_size(struct trace_probe *tp)
 849 {
 850         struct probe_entry_arg *earg = tp->entry_arg;
 851         int i, size = 0;
 852
 853         if (!earg)
 854                 return 0;
 855
 856         /*
 857          * earg->code[] array has an operation sequence which is run in
 858          * the entry handler.
 859          * The sequence stopped by FETCH_OP_END and each data stored in
 860          * the entry data buffer by FETCH_OP_ST_EDATA. The FETCH_OP_ST_EDATA
 861          * stores the data at the data buffer + its offset, and all data are
 862          * "unsigned long" size. The offset must be increased when a data is
 863          * stored. Thus we need to find the last FETCH_OP_ST_EDATA in the
 864          * code array.
 865          */
 866         for (i = 0; i < earg->size; i++) {
 867                 switch (earg->code[i].op) {
 868                 case FETCH_OP_END:
 869                         goto out;
 870                 case FETCH_OP_ST_EDATA:
 871                         size = earg->code[i].offset + sizeof(unsigned long);
 872                         break;
 873                 default:
 874                         break;
 875                 }
 876         }
 877 out:
 878         return size;
 879 }
 880
 881 void store_trace_entry_data(void *edata, struct trace_probe *tp, struct pt_regs *regs)
 882 {
 883         struct probe_entry_arg *earg = tp->entry_arg;
 884         unsigned long val = 0;
 885         int i;
 886
 887         if (!earg)
 888                 return;
 889
 890         for (i = 0; i < earg->size; i++) {
 891                 struct fetch_insn *code = &earg->code[i];
 892
 893                 switch (code->op) {
 894                 case FETCH_OP_ARG:
 895                         val = regs_get_kernel_argument(regs, code->param);
 896                         break;
 897                 case FETCH_OP_ST_EDATA:
 898                         *(unsigned long *)((unsigned long)edata + code->offset) = val;
 899                         break;
 900                 case FETCH_OP_END:
 901                         goto end;
 902                 default:
 903                         break;
 904                 }
 905         }
 906 end:
 907         return;
 908 }
 909 NOKPROBE_SYMBOL(store_trace_entry_data)
 910 #endif
 911
 912 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
 913
 914 /* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */
 915 static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
 916                             struct fetch_insn **pcode,
 917                             struct fetch_insn *end,
 918                             struct traceprobe_parse_context *ctx)
 919 {
 920         struct fetch_insn *code = *pcode;
 921         int err = TP_ERR_BAD_VAR;
 922         char *arg = orig_arg + 1;
 923         unsigned long param;
 924         int ret = 0;
 925         int len;
 926
 927         if (ctx->flags & TPARG_FL_TEVENT) {
 928                 if (code->data)
 929                         return -EFAULT;
 930                 ret = parse_trace_event_arg(arg, code, ctx);
 931                 if (!ret)
 932                         return 0;
 933                 if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
 934                         code->op = FETCH_OP_COMM;
 935                         return 0;
 936                 }
 937                 /* backward compatibility */
 938                 ctx->offset = 0;
 939                 goto inval;
 940         }
 941
 942         if (str_has_prefix(arg, "retval")) {
 943                 if (!(ctx->flags & TPARG_FL_RETURN)) {
 944                         err = TP_ERR_RETVAL_ON_PROBE;
 945                         goto inval;
 946                 }
 947                 if (!(ctx->flags & TPARG_FL_KERNEL) ||
 948                     !IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
 949                         code->op = FETCH_OP_RETVAL;
 950                         return 0;
 951                 }
 952                 return parse_btf_arg(orig_arg, pcode, end, ctx);
 953         }
 954
 955         len = str_has_prefix(arg, "stack");
 956         if (len) {
 957
 958                 if (arg[len] == '\0') {
 959                         code->op = FETCH_OP_STACKP;
 960                         return 0;
 961                 }
 962
 963                 if (isdigit(arg[len])) {
 964                         ret = kstrtoul(arg + len, 10, &param);
 965                         if (ret)
 966                                 goto inval;
 967
 968                         if ((ctx->flags & TPARG_FL_KERNEL) &&
 969                             param > PARAM_MAX_STACK) {
 970                                 err = TP_ERR_BAD_STACK_NUM;
 971                                 goto inval;
 972                         }
 973                         code->op = FETCH_OP_STACK;
 974                         code->param = (unsigned int)param;
 975                         return 0;
 976                 }
 977                 goto inval;
 978         }
 979
 980         if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
 981                 code->op = FETCH_OP_COMM;
 982                 return 0;
 983         }
 984
 985 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
 986         len = str_has_prefix(arg, "arg");
 987         if (len) {
 988                 ret = kstrtoul(arg + len, 10, &param);
 989                 if (ret)
 990                         goto inval;
 991
 992                 if (!param || param > PARAM_MAX_STACK) {
 993                         err = TP_ERR_BAD_ARG_NUM;
 994                         goto inval;
 995                 }
 996                 param--; /* argN starts from 1, but internal arg[N] starts from 0 */
 997
 998                 if (tparg_is_function_entry(ctx->flags)) {
 999                         code->op = FETCH_OP_ARG;
1000                         code->param = (unsigned int)param;
1001                         /*
1002                          * The tracepoint probe will probe a stub function, and the
1003                          * first parameter of the stub is a dummy and should be ignored.
1004                          */
1005                         if (ctx->flags & TPARG_FL_TPOINT)
1006                                 code->param++;
1007                 } else if (tparg_is_function_return(ctx->flags)) {
1008                         /* function entry argument access from return probe */
1009                         ret = __store_entry_arg(ctx->tp, param);
1010                         if (ret < 0)    /* This error should be an internal error */
1011                                 return ret;
1012
1013                         code->op = FETCH_OP_EDATA;
1014                         code->offset = ret;
1015                 } else {
1016                         err = TP_ERR_NOFENTRY_ARGS;
1017                         goto inval;
1018                 }
1019                 return 0;
1020         }
1021 #endif
1022
1023 inval:
1024         __trace_probe_log_err(ctx->offset, err);
1025         return -EINVAL;
1026 }
1027
1028 static int str_to_immediate(char *str, unsigned long *imm)
1029 {
1030         if (isdigit(str[0]))
1031                 return kstrtoul(str, 0, imm);
1032         else if (str[0] == '-')
1033                 return kstrtol(str, 0, (long *)imm);
1034         else if (str[0] == '+')
1035                 return kstrtol(str + 1, 0, (long *)imm);
1036         return -EINVAL;
1037 }
1038
1039 static int __parse_imm_string(char *str, char **pbuf, int offs)
1040 {
1041         size_t len = strlen(str);
1042
1043         if (str[len - 1] != '"') {
1044                 trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE);
1045                 return -EINVAL;
1046         }
1047         *pbuf = kstrndup(str, len - 1, GFP_KERNEL);
1048         if (!*pbuf)
1049                 return -ENOMEM;
1050         return 0;
1051 }
1052
1053 /* Recursive argument parser */
1054 static int
1055 parse_probe_arg(char *arg, const struct fetch_type *type,
1056                 struct fetch_insn **pcode, struct fetch_insn *end,
1057                 struct traceprobe_parse_context *ctx)
1058 {
1059         struct fetch_insn *code = *pcode;
1060         unsigned long param;
1061         int deref = FETCH_OP_DEREF;
1062         long offset = 0;
1063         char *tmp;
1064         int ret = 0;
1065
1066         switch (arg[0]) {
1067         case '$':
1068                 ret = parse_probe_vars(arg, type, pcode, end, ctx);
1069                 break;
1070
1071         case '%':       /* named register */
1072                 if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
1073                         /* eprobe and fprobe do not handle registers */
1074                         trace_probe_log_err(ctx->offset, BAD_VAR);
1075                         break;
1076                 }
1077                 ret = regs_query_register_offset(arg + 1);
1078                 if (ret >= 0) {
1079                         code->op = FETCH_OP_REG;
1080                         code->param = (unsigned int)ret;
1081                         ret = 0;
1082                 } else
1083                         trace_probe_log_err(ctx->offset, BAD_REG_NAME);
1084                 break;
1085
1086         case '@':       /* memory, file-offset or symbol */
1087                 if (isdigit(arg[1])) {
1088                         ret = kstrtoul(arg + 1, 0, &param);
1089                         if (ret) {
1090                                 trace_probe_log_err(ctx->offset, BAD_MEM_ADDR);
1091                                 break;
1092                         }
1093                         /* load address */
1094                         code->op = FETCH_OP_IMM;
1095                         code->immediate = param;
1096                 } else if (arg[1] == '+') {
1097                         /* kprobes don't support file offsets */
1098                         if (ctx->flags & TPARG_FL_KERNEL) {
1099                                 trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
1100                                 return -EINVAL;
1101                         }
1102                         ret = kstrtol(arg + 2, 0, &offset);
1103                         if (ret) {
1104                                 trace_probe_log_err(ctx->offset, BAD_FILE_OFFS);
1105                                 break;
1106                         }
1107
1108                         code->op = FETCH_OP_FOFFS;
1109                         code->immediate = (unsigned long)offset;  // imm64?
1110                 } else {
1111                         /* uprobes don't support symbols */
1112                         if (!(ctx->flags & TPARG_FL_KERNEL)) {
1113                                 trace_probe_log_err(ctx->offset, SYM_ON_UPROBE);
1114                                 return -EINVAL;
1115                         }
1116                         /* Preserve symbol for updating */
1117                         code->op = FETCH_NOP_SYMBOL;
1118                         code->data = kstrdup(arg + 1, GFP_KERNEL);
1119                         if (!code->data)
1120                                 return -ENOMEM;
1121                         if (++code == end) {
1122                                 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1123                                 return -EINVAL;
1124                         }
1125                         code->op = FETCH_OP_IMM;
1126                         code->immediate = 0;
1127                 }
1128                 /* These are fetching from memory */
1129                 if (++code == end) {
1130                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1131                         return -EINVAL;
1132                 }
1133                 *pcode = code;
1134                 code->op = FETCH_OP_DEREF;
1135                 code->offset = offset;
1136                 break;
1137
1138         case '+':       /* deref memory */
1139         case '-':
1140                 if (arg[1] == 'u') {
1141                         deref = FETCH_OP_UDEREF;
1142                         arg[1] = arg[0];
1143                         arg++;
1144                 }
1145                 if (arg[0] == '+')
1146                         arg++;  /* Skip '+', because kstrtol() rejects it. */
1147                 tmp = strchr(arg, '(');
1148                 if (!tmp) {
1149                         trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE);
1150                         return -EINVAL;
1151                 }
1152                 *tmp = '\0';
1153                 ret = kstrtol(arg, 0, &offset);
1154                 if (ret) {
1155                         trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS);
1156                         break;
1157                 }
1158                 ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
1159                 arg = tmp + 1;
1160                 tmp = strrchr(arg, ')');
1161                 if (!tmp) {
1162                         trace_probe_log_err(ctx->offset + strlen(arg),
1163                                             DEREF_OPEN_BRACE);
1164                         return -EINVAL;
1165                 } else {
1166                         const struct fetch_type *t2 = find_fetch_type(NULL, ctx->flags);
1167                         int cur_offs = ctx->offset;
1168
1169                         *tmp = '\0';
1170                         ret = parse_probe_arg(arg, t2, &code, end, ctx);
1171                         if (ret)
1172                                 break;
1173                         ctx->offset = cur_offs;
1174                         if (code->op == FETCH_OP_COMM ||
1175                             code->op == FETCH_OP_DATA) {
1176                                 trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
1177                                 return -EINVAL;
1178                         }
1179                         if (++code == end) {
1180                                 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1181                                 return -EINVAL;
1182                         }
1183                         *pcode = code;
1184
1185                         code->op = deref;
1186                         code->offset = offset;
1187                         /* Reset the last type if used */
1188                         ctx->last_type = NULL;
1189                 }
1190                 break;
1191         case '\\':      /* Immediate value */
1192                 if (arg[1] == '"') {    /* Immediate string */
1193                         ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2);
1194                         if (ret)
1195                                 break;
1196                         code->op = FETCH_OP_DATA;
1197                         code->data = tmp;
1198                 } else {
1199                         ret = str_to_immediate(arg + 1, &code->immediate);
1200                         if (ret)
1201                                 trace_probe_log_err(ctx->offset + 1, BAD_IMM);
1202                         else
1203                                 code->op = FETCH_OP_IMM;
1204                 }
1205                 break;
1206         default:
1207                 if (isalpha(arg[0]) || arg[0] == '_') { /* BTF variable */
1208                         if (!tparg_is_function_entry(ctx->flags) &&
1209                             !tparg_is_function_return(ctx->flags)) {
1210                                 trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1211                                 return -EINVAL;
1212                         }
1213                         ret = parse_btf_arg(arg, pcode, end, ctx);
1214                         break;
1215                 }
1216         }
1217         if (!ret && code->op == FETCH_OP_NOP) {
1218                 /* Parsed, but do not find fetch method */
1219                 trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
1220                 ret = -EINVAL;
1221         }
1222         return ret;
1223 }
1224
1225 /* Bitfield type needs to be parsed into a fetch function */
1226 static int __parse_bitfield_probe_arg(const char *bf,
1227                                       const struct fetch_type *t,
1228                                       struct fetch_insn **pcode)
1229 {
1230         struct fetch_insn *code = *pcode;
1231         unsigned long bw, bo;
1232         char *tail;
1233
1234         if (*bf != 'b')
1235                 return 0;
1236
1237         bw = simple_strtoul(bf + 1, &tail, 0);  /* Use simple one */
1238
1239         if (bw == 0 || *tail != '@')
1240                 return -EINVAL;
1241
1242         bf = tail + 1;
1243         bo = simple_strtoul(bf, &tail, 0);
1244
1245         if (tail == bf || *tail != '/')
1246                 return -EINVAL;
1247         code++;
1248         if (code->op != FETCH_OP_NOP)
1249                 return -EINVAL;
1250         *pcode = code;
1251
1252         code->op = FETCH_OP_MOD_BF;
1253         code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
1254         code->rshift = BYTES_TO_BITS(t->size) - bw;
1255         code->basesize = t->size;
1256
1257         return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
1258 }
1259
1260 /* Split type part from @arg and return it. */
1261 static char *parse_probe_arg_type(char *arg, struct probe_arg *parg,
1262                                   struct traceprobe_parse_context *ctx)
1263 {
1264         char *t = NULL, *t2, *t3;
1265         int offs;
1266
1267         t = strchr(arg, ':');
1268         if (t) {
1269                 *t++ = '\0';
1270                 t2 = strchr(t, '[');
1271                 if (t2) {
1272                         *t2++ = '\0';
1273                         t3 = strchr(t2, ']');
1274                         if (!t3) {
1275                                 offs = t2 + strlen(t2) - arg;
1276
1277                                 trace_probe_log_err(ctx->offset + offs,
1278                                                     ARRAY_NO_CLOSE);
1279                                 return ERR_PTR(-EINVAL);
1280                         } else if (t3[1] != '\0') {
1281                                 trace_probe_log_err(ctx->offset + t3 + 1 - arg,
1282                                                     BAD_ARRAY_SUFFIX);
1283                                 return ERR_PTR(-EINVAL);
1284                         }
1285                         *t3 = '\0';
1286                         if (kstrtouint(t2, 0, &parg->count) || !parg->count) {
1287                                 trace_probe_log_err(ctx->offset + t2 - arg,
1288                                                     BAD_ARRAY_NUM);
1289                                 return ERR_PTR(-EINVAL);
1290                         }
1291                         if (parg->count > MAX_ARRAY_LEN) {
1292                                 trace_probe_log_err(ctx->offset + t2 - arg,
1293                                                     ARRAY_TOO_BIG);
1294                                 return ERR_PTR(-EINVAL);
1295                         }
1296                 }
1297         }
1298         offs = t ? t - arg : 0;
1299
1300         /*
1301          * Since $comm and immediate string can not be dereferenced,
1302          * we can find those by strcmp. But ignore for eprobes.
1303          */
1304         if (!(ctx->flags & TPARG_FL_TEVENT) &&
1305             (strcmp(arg, "$comm") == 0 || strcmp(arg, "$COMM") == 0 ||
1306              strncmp(arg, "\\\"", 2) == 0)) {
1307                 /* The type of $comm must be "string", and not an array type. */
1308                 if (parg->count || (t && strcmp(t, "string"))) {
1309                         trace_probe_log_err(ctx->offset + offs, NEED_STRING_TYPE);
1310                         return ERR_PTR(-EINVAL);
1311                 }
1312                 parg->type = find_fetch_type("string", ctx->flags);
1313         } else
1314                 parg->type = find_fetch_type(t, ctx->flags);
1315
1316         if (!parg->type) {
1317                 trace_probe_log_err(ctx->offset + offs, BAD_TYPE);
1318                 return ERR_PTR(-EINVAL);
1319         }
1320
1321         return t;
1322 }
1323
1324 /* After parsing, adjust the fetch_insn according to the probe_arg */
1325 static int finalize_fetch_insn(struct fetch_insn *code,
1326                                struct probe_arg *parg,
1327                                char *type,
1328                                int type_offset,
1329                                struct traceprobe_parse_context *ctx)
1330 {
1331         struct fetch_insn *scode;
1332         int ret;
1333
1334         /* Store operation */
1335         if (parg->type->is_string) {
1336                 /* Check bad combination of the type and the last fetch_insn. */
1337                 if (!strcmp(parg->type->name, "symstr")) {
1338                         if (code->op != FETCH_OP_REG && code->op != FETCH_OP_STACK &&
1339                             code->op != FETCH_OP_RETVAL && code->op != FETCH_OP_ARG &&
1340                             code->op != FETCH_OP_DEREF && code->op != FETCH_OP_TP_ARG) {
1341                                 trace_probe_log_err(ctx->offset + type_offset,
1342                                                     BAD_SYMSTRING);
1343                                 return -EINVAL;
1344                         }
1345                 } else {
1346                         if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
1347                             code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
1348                             code->op != FETCH_OP_DATA && code->op != FETCH_OP_TP_ARG) {
1349                                 trace_probe_log_err(ctx->offset + type_offset,
1350                                                     BAD_STRING);
1351                                 return -EINVAL;
1352                         }
1353                 }
1354
1355                 if (!strcmp(parg->type->name, "symstr") ||
1356                     (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
1357                      code->op == FETCH_OP_DATA) || code->op == FETCH_OP_TP_ARG ||
1358                      parg->count) {
1359                         /*
1360                          * IMM, DATA and COMM is pointing actual address, those
1361                          * must be kept, and if parg->count != 0, this is an
1362                          * array of string pointers instead of string address
1363                          * itself.
1364                          * For the symstr, it doesn't need to dereference, thus
1365                          * it just get the value.
1366                          */
1367                         code++;
1368                         if (code->op != FETCH_OP_NOP) {
1369                                 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1370                                 return -EINVAL;
1371                         }
1372                 }
1373
1374                 /* If op == DEREF, replace it with STRING */
1375                 if (!strcmp(parg->type->name, "ustring") ||
1376                     code->op == FETCH_OP_UDEREF)
1377                         code->op = FETCH_OP_ST_USTRING;
1378                 else if (!strcmp(parg->type->name, "symstr"))
1379                         code->op = FETCH_OP_ST_SYMSTR;
1380                 else
1381                         code->op = FETCH_OP_ST_STRING;
1382                 code->size = parg->type->size;
1383                 parg->dynamic = true;
1384         } else if (code->op == FETCH_OP_DEREF) {
1385                 code->op = FETCH_OP_ST_MEM;
1386                 code->size = parg->type->size;
1387         } else if (code->op == FETCH_OP_UDEREF) {
1388                 code->op = FETCH_OP_ST_UMEM;
1389                 code->size = parg->type->size;
1390         } else {
1391                 code++;
1392                 if (code->op != FETCH_OP_NOP) {
1393                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1394                         return -E2BIG;
1395                 }
1396                 code->op = FETCH_OP_ST_RAW;
1397                 code->size = parg->type->size;
1398         }
1399
1400         /* Save storing fetch_insn. */
1401         scode = code;
1402
1403         /* Modify operation */
1404         if (type != NULL) {
1405                 /* Bitfield needs a special fetch_insn. */
1406                 ret = __parse_bitfield_probe_arg(type, parg->type, &code);
1407                 if (ret) {
1408                         trace_probe_log_err(ctx->offset + type_offset, BAD_BITFIELD);
1409                         return ret;
1410                 }
1411         } else if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1412                    ctx->last_type) {
1413                 /* If user not specified the type, try parsing BTF bitfield. */
1414                 ret = parse_btf_bitfield(&code, ctx);
1415                 if (ret)
1416                         return ret;
1417         }
1418
1419         /* Loop(Array) operation */
1420         if (parg->count) {
1421                 if (scode->op != FETCH_OP_ST_MEM &&
1422                     scode->op != FETCH_OP_ST_STRING &&
1423                     scode->op != FETCH_OP_ST_USTRING) {
1424                         trace_probe_log_err(ctx->offset + type_offset, BAD_STRING);
1425                         return -EINVAL;
1426                 }
1427                 code++;
1428                 if (code->op != FETCH_OP_NOP) {
1429                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1430                         return -E2BIG;
1431                 }
1432                 code->op = FETCH_OP_LP_ARRAY;
1433                 code->param = parg->count;
1434         }
1435
1436         /* Finalize the fetch_insn array. */
1437         code++;
1438         code->op = FETCH_OP_END;
1439
1440         return 0;
1441 }
1442
1443 /* String length checking wrapper */
1444 static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
1445                                            struct probe_arg *parg,
1446                                            struct traceprobe_parse_context *ctx)
1447 {
1448         struct fetch_insn *code, *tmp = NULL;
1449         char *type, *arg __free(kfree) = NULL;
1450         int ret, len;
1451
1452         len = strlen(argv);
1453         if (len > MAX_ARGSTR_LEN) {
1454                 trace_probe_log_err(ctx->offset, ARG_TOO_LONG);
1455                 return -E2BIG;
1456         } else if (len == 0) {
1457                 trace_probe_log_err(ctx->offset, NO_ARG_BODY);
1458                 return -EINVAL;
1459         }
1460
1461         arg = kstrdup(argv, GFP_KERNEL);
1462         if (!arg)
1463                 return -ENOMEM;
1464
1465         parg->comm = kstrdup(arg, GFP_KERNEL);
1466         if (!parg->comm)
1467                 return -ENOMEM;
1468
1469         type = parse_probe_arg_type(arg, parg, ctx);
1470         if (IS_ERR(type))
1471                 return PTR_ERR(type);
1472
1473         code = tmp = kcalloc(FETCH_INSN_MAX, sizeof(*code), GFP_KERNEL);
1474         if (!code)
1475                 return -ENOMEM;
1476         code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
1477
1478         ctx->last_type = NULL;
1479         ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
1480                               ctx);
1481         if (ret < 0)
1482                 goto fail;
1483
1484         /* Update storing type if BTF is available */
1485         if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1486             ctx->last_type) {
1487                 if (!type) {
1488                         parg->type = find_fetch_type_from_btf_type(ctx);
1489                 } else if (strstr(type, "string")) {
1490                         ret = check_prepare_btf_string_fetch(type, &code, ctx);
1491                         if (ret)
1492                                 goto fail;
1493                 }
1494         }
1495         parg->offset = *size;
1496         *size += parg->type->size * (parg->count ?: 1);
1497
1498         if (parg->count) {
1499                 len = strlen(parg->type->fmttype) + 6;
1500                 parg->fmt = kmalloc(len, GFP_KERNEL);
1501                 if (!parg->fmt) {
1502                         ret = -ENOMEM;
1503                         goto fail;
1504                 }
1505                 snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype,
1506                          parg->count);
1507         }
1508
1509         ret = finalize_fetch_insn(code, parg, type, type ? type - arg : 0, ctx);
1510         if (ret < 0)
1511                 goto fail;
1512
1513         for (; code < tmp + FETCH_INSN_MAX; code++)
1514                 if (code->op == FETCH_OP_END)
1515                         break;
1516         /* Shrink down the code buffer */
1517         parg->code = kcalloc(code - tmp + 1, sizeof(*code), GFP_KERNEL);
1518         if (!parg->code)
1519                 ret = -ENOMEM;
1520         else
1521                 memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
1522
1523 fail:
1524         if (ret < 0) {
1525                 for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
1526                         if (code->op == FETCH_NOP_SYMBOL ||
1527                             code->op == FETCH_OP_DATA)
1528                                 kfree(code->data);
1529         }
1530         kfree(tmp);
1531
1532         return ret;
1533 }
1534
1535 /* Return 1 if name is reserved or already used by another argument */
1536 static int traceprobe_conflict_field_name(const char *name,
1537                                           struct probe_arg *args, int narg)
1538 {
1539         int i;
1540
1541         for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
1542                 if (strcmp(reserved_field_names[i], name) == 0)
1543                         return 1;
1544
1545         for (i = 0; i < narg; i++)
1546                 if (strcmp(args[i].name, name) == 0)
1547                         return 1;
1548
1549         return 0;
1550 }
1551
1552 static char *generate_probe_arg_name(const char *arg, int idx)
1553 {
1554         char *name = NULL;
1555         const char *end;
1556
1557         /*
1558          * If argument name is omitted, try arg as a name (BTF variable)
1559          * or "argN".
1560          */
1561         if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
1562                 end = strchr(arg, ':');
1563                 if (!end)
1564                         end = arg + strlen(arg);
1565
1566                 name = kmemdup_nul(arg, end - arg, GFP_KERNEL);
1567                 if (!name || !is_good_name(name)) {
1568                         kfree(name);
1569                         name = NULL;
1570                 }
1571         }
1572
1573         if (!name)
1574                 name = kasprintf(GFP_KERNEL, "arg%d", idx + 1);
1575
1576         return name;
1577 }
1578
1579 int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg,
1580                                struct traceprobe_parse_context *ctx)
1581 {
1582         struct probe_arg *parg = &tp->args[i];
1583         const char *body;
1584
1585         ctx->tp = tp;
1586         body = strchr(arg, '=');
1587         if (body) {
1588                 if (body - arg > MAX_ARG_NAME_LEN) {
1589                         trace_probe_log_err(0, ARG_NAME_TOO_LONG);
1590                         return -EINVAL;
1591                 } else if (body == arg) {
1592                         trace_probe_log_err(0, NO_ARG_NAME);
1593                         return -EINVAL;
1594                 }
1595                 parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL);
1596                 body++;
1597         } else {
1598                 parg->name = generate_probe_arg_name(arg, i);
1599                 body = arg;
1600         }
1601         if (!parg->name)
1602                 return -ENOMEM;
1603
1604         if (!is_good_name(parg->name)) {
1605                 trace_probe_log_err(0, BAD_ARG_NAME);
1606                 return -EINVAL;
1607         }
1608         if (traceprobe_conflict_field_name(parg->name, tp->args, i)) {
1609                 trace_probe_log_err(0, USED_ARG_NAME);
1610                 return -EINVAL;
1611         }
1612         ctx->offset = body - arg;
1613         /* Parse fetch argument */
1614         return traceprobe_parse_probe_arg_body(body, &tp->size, parg, ctx);
1615 }
1616
1617 void traceprobe_free_probe_arg(struct probe_arg *arg)
1618 {
1619         struct fetch_insn *code = arg->code;
1620
1621         while (code && code->op != FETCH_OP_END) {
1622                 if (code->op == FETCH_NOP_SYMBOL ||
1623                     code->op == FETCH_OP_DATA)
1624                         kfree(code->data);
1625                 code++;
1626         }
1627         kfree(arg->code);
1628         kfree(arg->name);
1629         kfree(arg->comm);
1630         kfree(arg->fmt);
1631 }
1632
1633 static int argv_has_var_arg(int argc, const char *argv[], int *args_idx,
1634                             struct traceprobe_parse_context *ctx)
1635 {
1636         int i, found = 0;
1637
1638         for (i = 0; i < argc; i++)
1639                 if (str_has_prefix(argv[i], "$arg")) {
1640                         trace_probe_log_set_index(i + 2);
1641
1642                         if (!tparg_is_function_entry(ctx->flags) &&
1643                             !tparg_is_function_return(ctx->flags)) {
1644                                 trace_probe_log_err(0, NOFENTRY_ARGS);
1645                                 return -EINVAL;
1646                         }
1647
1648                         if (isdigit(argv[i][4])) {
1649                                 found = 1;
1650                                 continue;
1651                         }
1652
1653                         if (argv[i][4] != '*') {
1654                                 trace_probe_log_err(0, BAD_VAR);
1655                                 return -EINVAL;
1656                         }
1657
1658                         if (*args_idx >= 0 && *args_idx < argc) {
1659                                 trace_probe_log_err(0, DOUBLE_ARGS);
1660                                 return -EINVAL;
1661                         }
1662                         found = 1;
1663                         *args_idx = i;
1664                 }
1665
1666         return found;
1667 }
1668
1669 static int sprint_nth_btf_arg(int idx, const char *type,
1670                               char *buf, int bufsize,
1671                               struct traceprobe_parse_context *ctx)
1672 {
1673         const char *name;
1674         int ret;
1675
1676         if (idx >= ctx->nr_params) {
1677                 trace_probe_log_err(0, NO_BTFARG);
1678                 return -ENOENT;
1679         }
1680         name = btf_name_by_offset(ctx->btf, ctx->params[idx].name_off);
1681         if (!name) {
1682                 trace_probe_log_err(0, NO_BTF_ENTRY);
1683                 return -ENOENT;
1684         }
1685         ret = snprintf(buf, bufsize, "%s%s", name, type);
1686         if (ret >= bufsize) {
1687                 trace_probe_log_err(0, ARGS_2LONG);
1688                 return -E2BIG;
1689         }
1690         return ret;
1691 }
1692
1693 /* Return new_argv which must be freed after use */
1694 const char **traceprobe_expand_meta_args(int argc, const char *argv[],
1695                                          int *new_argc, char *buf, int bufsize,
1696                                          struct traceprobe_parse_context *ctx)
1697 {
1698         const struct btf_param *params = NULL;
1699         int i, j, n, used, ret, args_idx = -1;
1700         const char **new_argv __free(kfree) = NULL;
1701
1702         ret = argv_has_var_arg(argc, argv, &args_idx, ctx);
1703         if (ret < 0)
1704                 return ERR_PTR(ret);
1705
1706         if (!ret) {
1707                 *new_argc = argc;
1708                 return NULL;
1709         }
1710
1711         ret = query_btf_context(ctx);
1712         if (ret < 0 || ctx->nr_params == 0) {
1713                 if (args_idx != -1) {
1714                         /* $arg* requires BTF info */
1715                         trace_probe_log_err(0, NOSUP_BTFARG);
1716                         return (const char **)params;
1717                 }
1718                 *new_argc = argc;
1719                 return NULL;
1720         }
1721
1722         if (args_idx >= 0)
1723                 *new_argc = argc + ctx->nr_params - 1;
1724         else
1725                 *new_argc = argc;
1726
1727         new_argv = kcalloc(*new_argc, sizeof(char *), GFP_KERNEL);
1728         if (!new_argv)
1729                 return ERR_PTR(-ENOMEM);
1730
1731         used = 0;
1732         for (i = 0, j = 0; i < argc; i++) {
1733                 trace_probe_log_set_index(i + 2);
1734                 if (i == args_idx) {
1735                         for (n = 0; n < ctx->nr_params; n++) {
1736                                 ret = sprint_nth_btf_arg(n, "", buf + used,
1737                                                          bufsize - used, ctx);
1738                                 if (ret < 0)
1739                                         return ERR_PTR(ret);
1740
1741                                 new_argv[j++] = buf + used;
1742                                 used += ret + 1;
1743                         }
1744                         continue;
1745                 }
1746
1747                 if (str_has_prefix(argv[i], "$arg")) {
1748                         char *type = NULL;
1749
1750                         n = simple_strtoul(argv[i] + 4, &type, 10);
1751                         if (type && !(*type == ':' || *type == '\0')) {
1752                                 trace_probe_log_err(0, BAD_VAR);
1753                                 return ERR_PTR(-ENOENT);
1754                         }
1755                         /* Note: $argN starts from $arg1 */
1756                         ret = sprint_nth_btf_arg(n - 1, type, buf + used,
1757                                                  bufsize - used, ctx);
1758                         if (ret < 0)
1759                                 return ERR_PTR(ret);
1760                         new_argv[j++] = buf + used;
1761                         used += ret + 1;
1762                 } else
1763                         new_argv[j++] = argv[i];
1764         }
1765
1766         return_ptr(new_argv);
1767 }
1768
1769 /* @buf: *buf must be equal to NULL. Caller must to free *buf */
1770 int traceprobe_expand_dentry_args(int argc, const char *argv[], char **buf)
1771 {
1772         int i, used, ret;
1773         const int bufsize = MAX_DENTRY_ARGS_LEN;
1774         char *tmpbuf __free(kfree) = NULL;
1775
1776         if (*buf)
1777                 return -EINVAL;
1778
1779         used = 0;
1780         for (i = 0; i < argc; i++) {
1781                 char *tmp __free(kfree) = NULL;
1782                 char *equal;
1783                 size_t arg_len;
1784
1785                 if (!glob_match("*:%p[dD]", argv[i]))
1786                         continue;
1787
1788                 if (!tmpbuf) {
1789                         tmpbuf = kmalloc(bufsize, GFP_KERNEL);
1790                         if (!tmpbuf)
1791                                 return -ENOMEM;
1792                 }
1793
1794                 tmp = kstrdup(argv[i], GFP_KERNEL);
1795                 if (!tmp)
1796                         return -ENOMEM;
1797
1798                 equal = strchr(tmp, '=');
1799                 if (equal)
1800                         *equal = '\0';
1801                 arg_len = strlen(argv[i]);
1802                 tmp[arg_len - 4] = '\0';
1803                 if (argv[i][arg_len - 1] == 'd')
1804                         ret = snprintf(tmpbuf + used, bufsize - used,
1805                                        "%s%s+0x0(+0x%zx(%s)):string",
1806                                        equal ? tmp : "", equal ? "=" : "",
1807                                        offsetof(struct dentry, d_name.name),
1808                                        equal ? equal + 1 : tmp);
1809                 else
1810                         ret = snprintf(tmpbuf + used, bufsize - used,
1811                                        "%s%s+0x0(+0x%zx(+0x%zx(%s))):string",
1812                                        equal ? tmp : "", equal ? "=" : "",
1813                                        offsetof(struct dentry, d_name.name),
1814                                        offsetof(struct file, f_path.dentry),
1815                                        equal ? equal + 1 : tmp);
1816
1817                 if (ret >= bufsize - used)
1818                         return -ENOMEM;
1819                 argv[i] = tmpbuf + used;
1820                 used += ret + 1;
1821         }
1822
1823         *buf = no_free_ptr(tmpbuf);
1824         return 0;
1825 }
1826
1827 void traceprobe_finish_parse(struct traceprobe_parse_context *ctx)
1828 {
1829         clear_btf_context(ctx);
1830 }
1831
1832 int traceprobe_update_arg(struct probe_arg *arg)
1833 {
1834         struct fetch_insn *code = arg->code;
1835         long offset;
1836         char *tmp;
1837         char c;
1838         int ret = 0;
1839
1840         while (code && code->op != FETCH_OP_END) {
1841                 if (code->op == FETCH_NOP_SYMBOL) {
1842                         if (code[1].op != FETCH_OP_IMM)
1843                                 return -EINVAL;
1844
1845                         tmp = strpbrk(code->data, "+-");
1846                         if (tmp)
1847                                 c = *tmp;
1848                         ret = traceprobe_split_symbol_offset(code->data,
1849                                                              &offset);
1850                         if (ret)
1851                                 return ret;
1852
1853                         code[1].immediate =
1854                                 (unsigned long)kallsyms_lookup_name(code->data);
1855                         if (tmp)
1856                                 *tmp = c;
1857                         if (!code[1].immediate)
1858                                 return -ENOENT;
1859                         code[1].immediate += offset;
1860                 }
1861                 code++;
1862         }
1863         return 0;
1864 }
1865
1866 /* When len=0, we just calculate the needed length */
1867 #define LEN_OR_ZERO (len ? len - pos : 0)
1868 static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
1869                            enum probe_print_type ptype)
1870 {
1871         struct probe_arg *parg;
1872         int i, j;
1873         int pos = 0;
1874         const char *fmt, *arg;
1875
1876         switch (ptype) {
1877         case PROBE_PRINT_NORMAL:
1878                 fmt = "(%lx)";
1879                 arg = ", REC->" FIELD_STRING_IP;
1880                 break;
1881         case PROBE_PRINT_RETURN:
1882                 fmt = "(%lx <- %lx)";
1883                 arg = ", REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
1884                 break;
1885         case PROBE_PRINT_EVENT:
1886                 fmt = "";
1887                 arg = "";
1888                 break;
1889         default:
1890                 WARN_ON_ONCE(1);
1891                 return 0;
1892         }
1893
1894         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
1895
1896         for (i = 0; i < tp->nr_args; i++) {
1897                 parg = tp->args + i;
1898                 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name);
1899                 if (parg->count) {
1900                         pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s",
1901                                         parg->type->fmt);
1902                         for (j = 1; j < parg->count; j++)
1903                                 pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s",
1904                                                 parg->type->fmt);
1905                         pos += snprintf(buf + pos, LEN_OR_ZERO, "}");
1906                 } else
1907                         pos += snprintf(buf + pos, LEN_OR_ZERO, "%s",
1908                                         parg->type->fmt);
1909         }
1910
1911         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", arg);
1912
1913         for (i = 0; i < tp->nr_args; i++) {
1914                 parg = tp->args + i;
1915                 if (parg->count) {
1916                         if (parg->type->is_string)
1917                                 fmt = ", __get_str(%s[%d])";
1918                         else
1919                                 fmt = ", REC->%s[%d]";
1920                         for (j = 0; j < parg->count; j++)
1921                                 pos += snprintf(buf + pos, LEN_OR_ZERO,
1922                                                 fmt, parg->name, j);
1923                 } else {
1924                         if (parg->type->is_string)
1925                                 fmt = ", __get_str(%s)";
1926                         else
1927                                 fmt = ", REC->%s";
1928                         pos += snprintf(buf + pos, LEN_OR_ZERO,
1929                                         fmt, parg->name);
1930                 }
1931         }
1932
1933         /* return the length of print_fmt */
1934         return pos;
1935 }
1936 #undef LEN_OR_ZERO
1937
1938 int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype)
1939 {
1940         struct trace_event_call *call = trace_probe_event_call(tp);
1941         int len;
1942         char *print_fmt;
1943
1944         /* First: called with 0 length to calculate the needed length */
1945         len = __set_print_fmt(tp, NULL, 0, ptype);
1946         print_fmt = kmalloc(len + 1, GFP_KERNEL);
1947         if (!print_fmt)
1948                 return -ENOMEM;
1949
1950         /* Second: actually write the @print_fmt */
1951         __set_print_fmt(tp, print_fmt, len + 1, ptype);
1952         call->print_fmt = print_fmt;
1953
1954         return 0;
1955 }
1956
1957 int traceprobe_define_arg_fields(struct trace_event_call *event_call,
1958                                  size_t offset, struct trace_probe *tp)
1959 {
1960         int ret, i;
1961
1962         /* Set argument names as fields */
1963         for (i = 0; i < tp->nr_args; i++) {
1964                 struct probe_arg *parg = &tp->args[i];
1965                 const char *fmt = parg->type->fmttype;
1966                 int size = parg->type->size;
1967
1968                 if (parg->fmt)
1969                         fmt = parg->fmt;
1970                 if (parg->count)
1971                         size *= parg->count;
1972                 ret = trace_define_field(event_call, fmt, parg->name,
1973                                          offset + parg->offset, size,
1974                                          parg->type->is_signed,
1975                                          FILTER_OTHER);
1976                 if (ret)
1977                         return ret;
1978         }
1979         return 0;
1980 }
1981
1982 static void trace_probe_event_free(struct trace_probe_event *tpe)
1983 {
1984         kfree(tpe->class.system);
1985         kfree(tpe->call.name);
1986         kfree(tpe->call.print_fmt);
1987         kfree(tpe);
1988 }
1989
1990 int trace_probe_append(struct trace_probe *tp, struct trace_probe *to)
1991 {
1992         if (trace_probe_has_sibling(tp))
1993                 return -EBUSY;
1994
1995         list_del_init(&tp->list);
1996         trace_probe_event_free(tp->event);
1997
1998         tp->event = to->event;
1999         list_add_tail(&tp->list, trace_probe_probe_list(to));
2000
2001         return 0;
2002 }
2003
2004 void trace_probe_unlink(struct trace_probe *tp)
2005 {
2006         list_del_init(&tp->list);
2007         if (list_empty(trace_probe_probe_list(tp)))
2008                 trace_probe_event_free(tp->event);
2009         tp->event = NULL;
2010 }
2011
2012 void trace_probe_cleanup(struct trace_probe *tp)
2013 {
2014         int i;
2015
2016         for (i = 0; i < tp->nr_args; i++)
2017                 traceprobe_free_probe_arg(&tp->args[i]);
2018
2019         if (tp->entry_arg) {
2020                 kfree(tp->entry_arg->code);
2021                 kfree(tp->entry_arg);
2022                 tp->entry_arg = NULL;
2023         }
2024
2025         if (tp->event)
2026                 trace_probe_unlink(tp);
2027 }
2028
2029 int trace_probe_init(struct trace_probe *tp, const char *event,
2030                      const char *group, bool alloc_filter, int nargs)
2031 {
2032         struct trace_event_call *call;
2033         size_t size = sizeof(struct trace_probe_event);
2034         int ret = 0;
2035
2036         if (!event || !group)
2037                 return -EINVAL;
2038
2039         if (alloc_filter)
2040                 size += sizeof(struct trace_uprobe_filter);
2041
2042         tp->event = kzalloc(size, GFP_KERNEL);
2043         if (!tp->event)
2044                 return -ENOMEM;
2045
2046         INIT_LIST_HEAD(&tp->event->files);
2047         INIT_LIST_HEAD(&tp->event->class.fields);
2048         INIT_LIST_HEAD(&tp->event->probes);
2049         INIT_LIST_HEAD(&tp->list);
2050         list_add(&tp->list, &tp->event->probes);
2051
2052         call = trace_probe_event_call(tp);
2053         call->class = &tp->event->class;
2054         call->name = kstrdup(event, GFP_KERNEL);
2055         if (!call->name) {
2056                 ret = -ENOMEM;
2057                 goto error;
2058         }
2059
2060         tp->event->class.system = kstrdup(group, GFP_KERNEL);
2061         if (!tp->event->class.system) {
2062                 ret = -ENOMEM;
2063                 goto error;
2064         }
2065
2066         tp->nr_args = nargs;
2067         /* Make sure pointers in args[] are NULL */
2068         if (nargs)
2069                 memset(tp->args, 0, sizeof(tp->args[0]) * nargs);
2070
2071         return 0;
2072
2073 error:
2074         trace_probe_cleanup(tp);
2075         return ret;
2076 }
2077
2078 static struct trace_event_call *
2079 find_trace_event_call(const char *system, const char *event_name)
2080 {
2081         struct trace_event_call *tp_event;
2082         const char *name;
2083
2084         list_for_each_entry(tp_event, &ftrace_events, list) {
2085                 if (!tp_event->class->system ||
2086                     strcmp(system, tp_event->class->system))
2087                         continue;
2088                 name = trace_event_name(tp_event);
2089                 if (!name || strcmp(event_name, name))
2090                         continue;
2091                 return tp_event;
2092         }
2093
2094         return NULL;
2095 }
2096
2097 int trace_probe_register_event_call(struct trace_probe *tp)
2098 {
2099         struct trace_event_call *call = trace_probe_event_call(tp);
2100         int ret;
2101
2102         lockdep_assert_held(&event_mutex);
2103
2104         if (find_trace_event_call(trace_probe_group_name(tp),
2105                                   trace_probe_name(tp)))
2106                 return -EEXIST;
2107
2108         ret = register_trace_event(&call->event);
2109         if (!ret)
2110                 return -ENODEV;
2111
2112         ret = trace_add_event_call(call);
2113         if (ret)
2114                 unregister_trace_event(&call->event);
2115
2116         return ret;
2117 }
2118
2119 int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file)
2120 {
2121         struct event_file_link *link;
2122
2123         link = kmalloc(sizeof(*link), GFP_KERNEL);
2124         if (!link)
2125                 return -ENOMEM;
2126
2127         link->file = file;
2128         INIT_LIST_HEAD(&link->list);
2129         list_add_tail_rcu(&link->list, &tp->event->files);
2130         trace_probe_set_flag(tp, TP_FLAG_TRACE);
2131         return 0;
2132 }
2133
2134 struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp,
2135                                                   struct trace_event_file *file)
2136 {
2137         struct event_file_link *link;
2138
2139         trace_probe_for_each_link(link, tp) {
2140                 if (link->file == file)
2141                         return link;
2142         }
2143
2144         return NULL;
2145 }
2146
2147 int trace_probe_remove_file(struct trace_probe *tp,
2148                             struct trace_event_file *file)
2149 {
2150         struct event_file_link *link;
2151
2152         link = trace_probe_get_file_link(tp, file);
2153         if (!link)
2154                 return -ENOENT;
2155
2156         list_del_rcu(&link->list);
2157         kvfree_rcu_mightsleep(link);
2158
2159         if (list_empty(&tp->event->files))
2160                 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
2161
2162         return 0;
2163 }
2164
2165 /*
2166  * Return the smallest index of different type argument (start from 1).
2167  * If all argument types and name are same, return 0.
2168  */
2169 int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
2170 {
2171         int i;
2172
2173         /* In case of more arguments */
2174         if (a->nr_args < b->nr_args)
2175                 return a->nr_args + 1;
2176         if (a->nr_args > b->nr_args)
2177                 return b->nr_args + 1;
2178
2179         for (i = 0; i < a->nr_args; i++) {
2180                 if ((b->nr_args <= i) ||
2181                     ((a->args[i].type != b->args[i].type) ||
2182                      (a->args[i].count != b->args[i].count) ||
2183                      strcmp(a->args[i].name, b->args[i].name)))
2184                         return i + 1;
2185         }
2186
2187         return 0;
2188 }
2189
2190 bool trace_probe_match_command_args(struct trace_probe *tp,
2191                                     int argc, const char **argv)
2192 {
2193         char buf[MAX_ARGSTR_LEN + 1];
2194         int i;
2195
2196         if (tp->nr_args < argc)
2197                 return false;
2198
2199         for (i = 0; i < argc; i++) {
2200                 snprintf(buf, sizeof(buf), "%s=%s",
2201                          tp->args[i].name, tp->args[i].comm);
2202                 if (strcmp(buf, argv[i]))
2203                         return false;
2204         }
2205         return true;
2206 }
2207
2208 int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **))
2209 {
2210         int argc = 0, ret = 0;
2211         char **argv;
2212
2213         argv = argv_split(GFP_KERNEL, raw_command, &argc);
2214         if (!argv)
2215                 return -ENOMEM;
2216
2217         if (argc)
2218                 ret = createfn(argc, (const char **)argv);
2219
2220         argv_free(argv);
2221
2222         return ret;
2223 }
2224
2225 int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
2226                  u8 *data, void *field)
2227 {
2228         void *p;
2229         int i, j;
2230
2231         for (i = 0; i < nr_args; i++) {
2232                 struct probe_arg *a = args + i;
2233
2234                 trace_seq_printf(s, " %s=", a->name);
2235                 if (likely(!a->count)) {
2236                         if (!a->type->print(s, data + a->offset, field))
2237                                 return -ENOMEM;
2238                         continue;
2239                 }
2240                 trace_seq_putc(s, '{');
2241                 p = data + a->offset;
2242                 for (j = 0; j < a->count; j++) {
2243                         if (!a->type->print(s, p, field))
2244                                 return -ENOMEM;
2245                         trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
2246                         p += a->type->size;
2247                 }
2248         }
2249         return 0;
2250 }