kernel/trace/trace_uprobe.c

   1 /*
   2  * uprobes-based tracing events
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  16  *
  17  * Copyright (C) IBM Corporation, 2010-2012
  18  * Author:      Srikar Dronamraju <srikar@linux.vnet.ibm.com>
  19  */
  20
  21 #include <linux/module.h>
  22 #include <linux/uaccess.h>
  23 #include <linux/uprobes.h>
  24 #include <linux/namei.h>
  25 #include <linux/string.h>
  26
  27 #include "trace_probe.h"
  28
  29 #define UPROBE_EVENT_SYSTEM     "uprobes"
  30
  31 struct uprobe_trace_entry_head {
  32         struct trace_entry      ent;
  33         unsigned long           vaddr[];
  34 };
  35
  36 #define SIZEOF_TRACE_ENTRY(is_return)                   \
  37         (sizeof(struct uprobe_trace_entry_head) +       \
  38          sizeof(unsigned long) * (is_return ? 2 : 1))
  39
  40 #define DATAOF_TRACE_ENTRY(entry, is_return)            \
  41         ((void*)(entry) + SIZEOF_TRACE_ENTRY(is_return))
  42
  43 struct trace_uprobe_filter {
  44         rwlock_t                rwlock;
  45         int                     nr_systemwide;
  46         struct list_head        perf_events;
  47 };
  48
  49 /*
  50  * uprobe event core functions
  51  */
  52 struct trace_uprobe {
  53         struct list_head                list;
  54         struct trace_uprobe_filter      filter;
  55         struct uprobe_consumer          consumer;
  56         struct inode                    *inode;
  57         char                            *filename;
  58         unsigned long                   offset;
  59         unsigned long                   nhit;
  60         struct trace_probe              tp;
  61 };
  62
  63 #define SIZEOF_TRACE_UPROBE(n)                          \
  64         (offsetof(struct trace_uprobe, tp.args) +       \
  65         (sizeof(struct probe_arg) * (n)))
  66
  67 static int register_uprobe_event(struct trace_uprobe *tu);
  68 static int unregister_uprobe_event(struct trace_uprobe *tu);
  69
  70 static DEFINE_MUTEX(uprobe_lock);
  71 static LIST_HEAD(uprobe_list);
  72
  73 static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs);
  74 static int uretprobe_dispatcher(struct uprobe_consumer *con,
  75                                 unsigned long func, struct pt_regs *regs);
  76
  77 /* Fetch type information table */
  78 const struct fetch_type uprobes_fetch_type_table[] = {
  79         /* Special types */
  80         [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
  81                                         sizeof(u32), 1, "__data_loc char[]"),
  82         [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
  83                                         string_size, sizeof(u32), 0, "u32"),
  84         /* Basic types */
  85         ASSIGN_FETCH_TYPE(u8,  u8,  0),
  86         ASSIGN_FETCH_TYPE(u16, u16, 0),
  87         ASSIGN_FETCH_TYPE(u32, u32, 0),
  88         ASSIGN_FETCH_TYPE(u64, u64, 0),
  89         ASSIGN_FETCH_TYPE(s8,  u8,  1),
  90         ASSIGN_FETCH_TYPE(s16, u16, 1),
  91         ASSIGN_FETCH_TYPE(s32, u32, 1),
  92         ASSIGN_FETCH_TYPE(s64, u64, 1),
  93
  94         ASSIGN_FETCH_TYPE_END
  95 };
  96
  97 static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter)
  98 {
  99         rwlock_init(&filter->rwlock);
 100         filter->nr_systemwide = 0;
 101         INIT_LIST_HEAD(&filter->perf_events);
 102 }
 103
 104 static inline bool uprobe_filter_is_empty(struct trace_uprobe_filter *filter)
 105 {
 106         return !filter->nr_systemwide && list_empty(&filter->perf_events);
 107 }
 108
 109 static inline bool is_ret_probe(struct trace_uprobe *tu)
 110 {
 111         return tu->consumer.ret_handler != NULL;
 112 }
 113
 114 /*
 115  * Allocate new trace_uprobe and initialize it (including uprobes).
 116  */
 117 static struct trace_uprobe *
 118 alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)
 119 {
 120         struct trace_uprobe *tu;
 121
 122         if (!event || !is_good_name(event))
 123                 return ERR_PTR(-EINVAL);
 124
 125         if (!group || !is_good_name(group))
 126                 return ERR_PTR(-EINVAL);
 127
 128         tu = kzalloc(SIZEOF_TRACE_UPROBE(nargs), GFP_KERNEL);
 129         if (!tu)
 130                 return ERR_PTR(-ENOMEM);
 131
 132         tu->tp.call.class = &tu->tp.class;
 133         tu->tp.call.name = kstrdup(event, GFP_KERNEL);
 134         if (!tu->tp.call.name)
 135                 goto error;
 136
 137         tu->tp.class.system = kstrdup(group, GFP_KERNEL);
 138         if (!tu->tp.class.system)
 139                 goto error;
 140
 141         INIT_LIST_HEAD(&tu->list);
 142         tu->consumer.handler = uprobe_dispatcher;
 143         if (is_ret)
 144                 tu->consumer.ret_handler = uretprobe_dispatcher;
 145         init_trace_uprobe_filter(&tu->filter);
 146         tu->tp.call.flags |= TRACE_EVENT_FL_USE_CALL_FILTER;
 147         return tu;
 148
 149 error:
 150         kfree(tu->tp.call.name);
 151         kfree(tu);
 152
 153         return ERR_PTR(-ENOMEM);
 154 }
 155
 156 static void free_trace_uprobe(struct trace_uprobe *tu)
 157 {
 158         int i;
 159
 160         for (i = 0; i < tu->tp.nr_args; i++)
 161                 traceprobe_free_probe_arg(&tu->tp.args[i]);
 162
 163         iput(tu->inode);
 164         kfree(tu->tp.call.class->system);
 165         kfree(tu->tp.call.name);
 166         kfree(tu->filename);
 167         kfree(tu);
 168 }
 169
 170 static struct trace_uprobe *find_probe_event(const char *event, const char *group)
 171 {
 172         struct trace_uprobe *tu;
 173
 174         list_for_each_entry(tu, &uprobe_list, list)
 175                 if (strcmp(tu->tp.call.name, event) == 0 &&
 176                     strcmp(tu->tp.call.class->system, group) == 0)
 177                         return tu;
 178
 179         return NULL;
 180 }
 181
 182 /* Unregister a trace_uprobe and probe_event: call with locking uprobe_lock */
 183 static int unregister_trace_uprobe(struct trace_uprobe *tu)
 184 {
 185         int ret;
 186
 187         ret = unregister_uprobe_event(tu);
 188         if (ret)
 189                 return ret;
 190
 191         list_del(&tu->list);
 192         free_trace_uprobe(tu);
 193         return 0;
 194 }
 195
 196 /* Register a trace_uprobe and probe_event */
 197 static int register_trace_uprobe(struct trace_uprobe *tu)
 198 {
 199         struct trace_uprobe *old_tu;
 200         int ret;
 201
 202         mutex_lock(&uprobe_lock);
 203
 204         /* register as an event */
 205         old_tu = find_probe_event(tu->tp.call.name, tu->tp.call.class->system);
 206         if (old_tu) {
 207                 /* delete old event */
 208                 ret = unregister_trace_uprobe(old_tu);
 209                 if (ret)
 210                         goto end;
 211         }
 212
 213         ret = register_uprobe_event(tu);
 214         if (ret) {
 215                 pr_warning("Failed to register probe event(%d)\n", ret);
 216                 goto end;
 217         }
 218
 219         list_add_tail(&tu->list, &uprobe_list);
 220
 221 end:
 222         mutex_unlock(&uprobe_lock);
 223
 224         return ret;
 225 }
 226
 227 /*
 228  * Argument syntax:
 229  *  - Add uprobe: p|r[:[GRP/]EVENT] PATH:OFFSET [FETCHARGS]
 230  *
 231  *  - Remove uprobe: -:[GRP/]EVENT
 232  */
 233 static int create_trace_uprobe(int argc, char **argv)
 234 {
 235         struct trace_uprobe *tu;
 236         struct inode *inode;
 237         char *arg, *event, *group, *filename;
 238         char buf[MAX_EVENT_NAME_LEN];
 239         struct path path;
 240         unsigned long offset;
 241         bool is_delete, is_return;
 242         int i, ret;
 243
 244         inode = NULL;
 245         ret = 0;
 246         is_delete = false;
 247         is_return = false;
 248         event = NULL;
 249         group = NULL;
 250
 251         /* argc must be >= 1 */
 252         if (argv[0][0] == '-')
 253                 is_delete = true;
 254         else if (argv[0][0] == 'r')
 255                 is_return = true;
 256         else if (argv[0][0] != 'p') {
 257                 pr_info("Probe definition must be started with 'p', 'r' or '-'.\n");
 258                 return -EINVAL;
 259         }
 260
 261         if (argv[0][1] == ':') {
 262                 event = &argv[0][2];
 263                 arg = strchr(event, '/');
 264
 265                 if (arg) {
 266                         group = event;
 267                         event = arg + 1;
 268                         event[-1] = '\0';
 269
 270                         if (strlen(group) == 0) {
 271                                 pr_info("Group name is not specified\n");
 272                                 return -EINVAL;
 273                         }
 274                 }
 275                 if (strlen(event) == 0) {
 276                         pr_info("Event name is not specified\n");
 277                         return -EINVAL;
 278                 }
 279         }
 280         if (!group)
 281                 group = UPROBE_EVENT_SYSTEM;
 282
 283         if (is_delete) {
 284                 int ret;
 285
 286                 if (!event) {
 287                         pr_info("Delete command needs an event name.\n");
 288                         return -EINVAL;
 289                 }
 290                 mutex_lock(&uprobe_lock);
 291                 tu = find_probe_event(event, group);
 292
 293                 if (!tu) {
 294                         mutex_unlock(&uprobe_lock);
 295                         pr_info("Event %s/%s doesn't exist.\n", group, event);
 296                         return -ENOENT;
 297                 }
 298                 /* delete an event */
 299                 ret = unregister_trace_uprobe(tu);
 300                 mutex_unlock(&uprobe_lock);
 301                 return ret;
 302         }
 303
 304         if (argc < 2) {
 305                 pr_info("Probe point is not specified.\n");
 306                 return -EINVAL;
 307         }
 308         if (isdigit(argv[1][0])) {
 309                 pr_info("probe point must be have a filename.\n");
 310                 return -EINVAL;
 311         }
 312         arg = strchr(argv[1], ':');
 313         if (!arg) {
 314                 ret = -EINVAL;
 315                 goto fail_address_parse;
 316         }
 317
 318         *arg++ = '\0';
 319         filename = argv[1];
 320         ret = kern_path(filename, LOOKUP_FOLLOW, &path);
 321         if (ret)
 322                 goto fail_address_parse;
 323
 324         inode = igrab(path.dentry->d_inode);
 325         path_put(&path);
 326
 327         if (!inode || !S_ISREG(inode->i_mode)) {
 328                 ret = -EINVAL;
 329                 goto fail_address_parse;
 330         }
 331
 332         ret = kstrtoul(arg, 0, &offset);
 333         if (ret)
 334                 goto fail_address_parse;
 335
 336         argc -= 2;
 337         argv += 2;
 338
 339         /* setup a probe */
 340         if (!event) {
 341                 char *tail;
 342                 char *ptr;
 343
 344                 tail = kstrdup(kbasename(filename), GFP_KERNEL);
 345                 if (!tail) {
 346                         ret = -ENOMEM;
 347                         goto fail_address_parse;
 348                 }
 349
 350                 ptr = strpbrk(tail, ".-_");
 351                 if (ptr)
 352                         *ptr = '\0';
 353
 354                 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset);
 355                 event = buf;
 356                 kfree(tail);
 357         }
 358
 359         tu = alloc_trace_uprobe(group, event, argc, is_return);
 360         if (IS_ERR(tu)) {
 361                 pr_info("Failed to allocate trace_uprobe.(%d)\n", (int)PTR_ERR(tu));
 362                 ret = PTR_ERR(tu);
 363                 goto fail_address_parse;
 364         }
 365         tu->offset = offset;
 366         tu->inode = inode;
 367         tu->filename = kstrdup(filename, GFP_KERNEL);
 368
 369         if (!tu->filename) {
 370                 pr_info("Failed to allocate filename.\n");
 371                 ret = -ENOMEM;
 372                 goto error;
 373         }
 374
 375         /* parse arguments */
 376         ret = 0;
 377         for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
 378                 struct probe_arg *parg = &tu->tp.args[i];
 379
 380                 /* Increment count for freeing args in error case */
 381                 tu->tp.nr_args++;
 382
 383                 /* Parse argument name */
 384                 arg = strchr(argv[i], '=');
 385                 if (arg) {
 386                         *arg++ = '\0';
 387                         parg->name = kstrdup(argv[i], GFP_KERNEL);
 388                 } else {
 389                         arg = argv[i];
 390                         /* If argument name is omitted, set "argN" */
 391                         snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
 392                         parg->name = kstrdup(buf, GFP_KERNEL);
 393                 }
 394
 395                 if (!parg->name) {
 396                         pr_info("Failed to allocate argument[%d] name.\n", i);
 397                         ret = -ENOMEM;
 398                         goto error;
 399                 }
 400
 401                 if (!is_good_name(parg->name)) {
 402                         pr_info("Invalid argument[%d] name: %s\n", i, parg->name);
 403                         ret = -EINVAL;
 404                         goto error;
 405                 }
 406
 407                 if (traceprobe_conflict_field_name(parg->name, tu->tp.args, i)) {
 408                         pr_info("Argument[%d] name '%s' conflicts with "
 409                                 "another field.\n", i, argv[i]);
 410                         ret = -EINVAL;
 411                         goto error;
 412                 }
 413
 414                 /* Parse fetch argument */
 415                 ret = traceprobe_parse_probe_arg(arg, &tu->tp.size, parg,
 416                                                  false, false);
 417                 if (ret) {
 418                         pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
 419                         goto error;
 420                 }
 421         }
 422
 423         ret = register_trace_uprobe(tu);
 424         if (ret)
 425                 goto error;
 426         return 0;
 427
 428 error:
 429         free_trace_uprobe(tu);
 430         return ret;
 431
 432 fail_address_parse:
 433         if (inode)
 434                 iput(inode);
 435
 436         pr_info("Failed to parse address or file.\n");
 437
 438         return ret;
 439 }
 440
 441 static int cleanup_all_probes(void)
 442 {
 443         struct trace_uprobe *tu;
 444         int ret = 0;
 445
 446         mutex_lock(&uprobe_lock);
 447         while (!list_empty(&uprobe_list)) {
 448                 tu = list_entry(uprobe_list.next, struct trace_uprobe, list);
 449                 ret = unregister_trace_uprobe(tu);
 450                 if (ret)
 451                         break;
 452         }
 453         mutex_unlock(&uprobe_lock);
 454         return ret;
 455 }
 456
 457 /* Probes listing interfaces */
 458 static void *probes_seq_start(struct seq_file *m, loff_t *pos)
 459 {
 460         mutex_lock(&uprobe_lock);
 461         return seq_list_start(&uprobe_list, *pos);
 462 }
 463
 464 static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)
 465 {
 466         return seq_list_next(v, &uprobe_list, pos);
 467 }
 468
 469 static void probes_seq_stop(struct seq_file *m, void *v)
 470 {
 471         mutex_unlock(&uprobe_lock);
 472 }
 473
 474 static int probes_seq_show(struct seq_file *m, void *v)
 475 {
 476         struct trace_uprobe *tu = v;
 477         char c = is_ret_probe(tu) ? 'r' : 'p';
 478         int i;
 479
 480         seq_printf(m, "%c:%s/%s", c, tu->tp.call.class->system, tu->tp.call.name);
 481         seq_printf(m, " %s:0x%p", tu->filename, (void *)tu->offset);
 482
 483         for (i = 0; i < tu->tp.nr_args; i++)
 484                 seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
 485
 486         seq_printf(m, "\n");
 487         return 0;
 488 }
 489
 490 static const struct seq_operations probes_seq_op = {
 491         .start  = probes_seq_start,
 492         .next   = probes_seq_next,
 493         .stop   = probes_seq_stop,
 494         .show   = probes_seq_show
 495 };
 496
 497 static int probes_open(struct inode *inode, struct file *file)
 498 {
 499         int ret;
 500
 501         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
 502                 ret = cleanup_all_probes();
 503                 if (ret)
 504                         return ret;
 505         }
 506
 507         return seq_open(file, &probes_seq_op);
 508 }
 509
 510 static ssize_t probes_write(struct file *file, const char __user *buffer,
 511                             size_t count, loff_t *ppos)
 512 {
 513         return traceprobe_probes_write(file, buffer, count, ppos, create_trace_uprobe);
 514 }
 515
 516 static const struct file_operations uprobe_events_ops = {
 517         .owner          = THIS_MODULE,
 518         .open           = probes_open,
 519         .read           = seq_read,
 520         .llseek         = seq_lseek,
 521         .release        = seq_release,
 522         .write          = probes_write,
 523 };
 524
 525 /* Probes profiling interfaces */
 526 static int probes_profile_seq_show(struct seq_file *m, void *v)
 527 {
 528         struct trace_uprobe *tu = v;
 529
 530         seq_printf(m, "  %s %-44s %15lu\n", tu->filename, tu->tp.call.name, tu->nhit);
 531         return 0;
 532 }
 533
 534 static const struct seq_operations profile_seq_op = {
 535         .start  = probes_seq_start,
 536         .next   = probes_seq_next,
 537         .stop   = probes_seq_stop,
 538         .show   = probes_profile_seq_show
 539 };
 540
 541 static int profile_open(struct inode *inode, struct file *file)
 542 {
 543         return seq_open(file, &profile_seq_op);
 544 }
 545
 546 static const struct file_operations uprobe_profile_ops = {
 547         .owner          = THIS_MODULE,
 548         .open           = profile_open,
 549         .read           = seq_read,
 550         .llseek         = seq_lseek,
 551         .release        = seq_release,
 552 };
 553
 554 static void uprobe_trace_print(struct trace_uprobe *tu,
 555                                 unsigned long func, struct pt_regs *regs)
 556 {
 557         struct uprobe_trace_entry_head *entry;
 558         struct ring_buffer_event *event;
 559         struct ring_buffer *buffer;
 560         void *data;
 561         int size, i;
 562         struct ftrace_event_call *call = &tu->tp.call;
 563
 564         size = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
 565         event = trace_current_buffer_lock_reserve(&buffer, call->event.type,
 566                                                   size + tu->tp.size, 0, 0);
 567         if (!event)
 568                 return;
 569
 570         entry = ring_buffer_event_data(event);
 571         if (is_ret_probe(tu)) {
 572                 entry->vaddr[0] = func;
 573                 entry->vaddr[1] = instruction_pointer(regs);
 574                 data = DATAOF_TRACE_ENTRY(entry, true);
 575         } else {
 576                 entry->vaddr[0] = instruction_pointer(regs);
 577                 data = DATAOF_TRACE_ENTRY(entry, false);
 578         }
 579
 580         for (i = 0; i < tu->tp.nr_args; i++) {
 581                 call_fetch(&tu->tp.args[i].fetch, regs,
 582                            data + tu->tp.args[i].offset);
 583         }
 584
 585         if (!call_filter_check_discard(call, entry, buffer, event))
 586                 trace_buffer_unlock_commit(buffer, event, 0, 0);
 587 }
 588
 589 /* uprobe handler */
 590 static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs)
 591 {
 592         if (!is_ret_probe(tu))
 593                 uprobe_trace_print(tu, 0, regs);
 594         return 0;
 595 }
 596
 597 static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func,
 598                                 struct pt_regs *regs)
 599 {
 600         uprobe_trace_print(tu, func, regs);
 601 }
 602
 603 /* Event entry printers */
 604 static enum print_line_t
 605 print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event)
 606 {
 607         struct uprobe_trace_entry_head *entry;
 608         struct trace_seq *s = &iter->seq;
 609         struct trace_uprobe *tu;
 610         u8 *data;
 611         int i;
 612
 613         entry = (struct uprobe_trace_entry_head *)iter->ent;
 614         tu = container_of(event, struct trace_uprobe, tp.call.event);
 615
 616         if (is_ret_probe(tu)) {
 617                 if (!trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)", tu->tp.call.name,
 618                                         entry->vaddr[1], entry->vaddr[0]))
 619                         goto partial;
 620                 data = DATAOF_TRACE_ENTRY(entry, true);
 621         } else {
 622                 if (!trace_seq_printf(s, "%s: (0x%lx)", tu->tp.call.name,
 623                                         entry->vaddr[0]))
 624                         goto partial;
 625                 data = DATAOF_TRACE_ENTRY(entry, false);
 626         }
 627
 628         for (i = 0; i < tu->tp.nr_args; i++) {
 629                 struct probe_arg *parg = &tu->tp.args[i];
 630
 631                 if (!parg->type->print(s, parg->name, data + parg->offset, entry))
 632                         goto partial;
 633         }
 634
 635         if (trace_seq_puts(s, "\n"))
 636                 return TRACE_TYPE_HANDLED;
 637
 638 partial:
 639         return TRACE_TYPE_PARTIAL_LINE;
 640 }
 641
 642 typedef bool (*filter_func_t)(struct uprobe_consumer *self,
 643                                 enum uprobe_filter_ctx ctx,
 644                                 struct mm_struct *mm);
 645
 646 static int
 647 probe_event_enable(struct trace_uprobe *tu, int flag, filter_func_t filter)
 648 {
 649         int ret = 0;
 650
 651         if (trace_probe_is_enabled(&tu->tp))
 652                 return -EINTR;
 653
 654         WARN_ON(!uprobe_filter_is_empty(&tu->filter));
 655
 656         tu->tp.flags |= flag;
 657         tu->consumer.filter = filter;
 658         ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
 659         if (ret)
 660                 tu->tp.flags &= ~flag;
 661
 662         return ret;
 663 }
 664
 665 static void probe_event_disable(struct trace_uprobe *tu, int flag)
 666 {
 667         if (!trace_probe_is_enabled(&tu->tp))
 668                 return;
 669
 670         WARN_ON(!uprobe_filter_is_empty(&tu->filter));
 671
 672         uprobe_unregister(tu->inode, tu->offset, &tu->consumer);
 673         tu->tp.flags &= ~flag;
 674 }
 675
 676 static int uprobe_event_define_fields(struct ftrace_event_call *event_call)
 677 {
 678         int ret, i, size;
 679         struct uprobe_trace_entry_head field;
 680         struct trace_uprobe *tu = event_call->data;
 681
 682         if (is_ret_probe(tu)) {
 683                 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_FUNC, 0);
 684                 DEFINE_FIELD(unsigned long, vaddr[1], FIELD_STRING_RETIP, 0);
 685                 size = SIZEOF_TRACE_ENTRY(true);
 686         } else {
 687                 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0);
 688                 size = SIZEOF_TRACE_ENTRY(false);
 689         }
 690         /* Set argument names as fields */
 691         for (i = 0; i < tu->tp.nr_args; i++) {
 692                 struct probe_arg *parg = &tu->tp.args[i];
 693
 694                 ret = trace_define_field(event_call, parg->type->fmttype,
 695                                          parg->name, size + parg->offset,
 696                                          parg->type->size, parg->type->is_signed,
 697                                          FILTER_OTHER);
 698
 699                 if (ret)
 700                         return ret;
 701         }
 702         return 0;
 703 }
 704
 705 #ifdef CONFIG_PERF_EVENTS
 706 static bool
 707 __uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm)
 708 {
 709         struct perf_event *event;
 710
 711         if (filter->nr_systemwide)
 712                 return true;
 713
 714         list_for_each_entry(event, &filter->perf_events, hw.tp_list) {
 715                 if (event->hw.tp_target->mm == mm)
 716                         return true;
 717         }
 718
 719         return false;
 720 }
 721
 722 static inline bool
 723 uprobe_filter_event(struct trace_uprobe *tu, struct perf_event *event)
 724 {
 725         return __uprobe_perf_filter(&tu->filter, event->hw.tp_target->mm);
 726 }
 727
 728 static int uprobe_perf_open(struct trace_uprobe *tu, struct perf_event *event)
 729 {
 730         bool done;
 731
 732         write_lock(&tu->filter.rwlock);
 733         if (event->hw.tp_target) {
 734                 /*
 735                  * event->parent != NULL means copy_process(), we can avoid
 736                  * uprobe_apply(). current->mm must be probed and we can rely
 737                  * on dup_mmap() which preserves the already installed bp's.
 738                  *
 739                  * attr.enable_on_exec means that exec/mmap will install the
 740                  * breakpoints we need.
 741                  */
 742                 done = tu->filter.nr_systemwide ||
 743                         event->parent || event->attr.enable_on_exec ||
 744                         uprobe_filter_event(tu, event);
 745                 list_add(&event->hw.tp_list, &tu->filter.perf_events);
 746         } else {
 747                 done = tu->filter.nr_systemwide;
 748                 tu->filter.nr_systemwide++;
 749         }
 750         write_unlock(&tu->filter.rwlock);
 751
 752         if (!done)
 753                 uprobe_apply(tu->inode, tu->offset, &tu->consumer, true);
 754
 755         return 0;
 756 }
 757
 758 static int uprobe_perf_close(struct trace_uprobe *tu, struct perf_event *event)
 759 {
 760         bool done;
 761
 762         write_lock(&tu->filter.rwlock);
 763         if (event->hw.tp_target) {
 764                 list_del(&event->hw.tp_list);
 765                 done = tu->filter.nr_systemwide ||
 766                         (event->hw.tp_target->flags & PF_EXITING) ||
 767                         uprobe_filter_event(tu, event);
 768         } else {
 769                 tu->filter.nr_systemwide--;
 770                 done = tu->filter.nr_systemwide;
 771         }
 772         write_unlock(&tu->filter.rwlock);
 773
 774         if (!done)
 775                 uprobe_apply(tu->inode, tu->offset, &tu->consumer, false);
 776
 777         return 0;
 778 }
 779
 780 static bool uprobe_perf_filter(struct uprobe_consumer *uc,
 781                                 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
 782 {
 783         struct trace_uprobe *tu;
 784         int ret;
 785
 786         tu = container_of(uc, struct trace_uprobe, consumer);
 787         read_lock(&tu->filter.rwlock);
 788         ret = __uprobe_perf_filter(&tu->filter, mm);
 789         read_unlock(&tu->filter.rwlock);
 790
 791         return ret;
 792 }
 793
 794 static void uprobe_perf_print(struct trace_uprobe *tu,
 795                                 unsigned long func, struct pt_regs *regs)
 796 {
 797         struct ftrace_event_call *call = &tu->tp.call;
 798         struct uprobe_trace_entry_head *entry;
 799         struct hlist_head *head;
 800         void *data;
 801         int size, rctx, i;
 802
 803         size = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
 804         size = ALIGN(size + tu->tp.size + sizeof(u32), sizeof(u64)) - sizeof(u32);
 805
 806         preempt_disable();
 807         head = this_cpu_ptr(call->perf_events);
 808         if (hlist_empty(head))
 809                 goto out;
 810
 811         entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
 812         if (!entry)
 813                 goto out;
 814
 815         if (is_ret_probe(tu)) {
 816                 entry->vaddr[0] = func;
 817                 entry->vaddr[1] = instruction_pointer(regs);
 818                 data = DATAOF_TRACE_ENTRY(entry, true);
 819         } else {
 820                 entry->vaddr[0] = instruction_pointer(regs);
 821                 data = DATAOF_TRACE_ENTRY(entry, false);
 822         }
 823
 824         for (i = 0; i < tu->tp.nr_args; i++) {
 825                 struct probe_arg *parg = &tu->tp.args[i];
 826
 827                 call_fetch(&parg->fetch, regs, data + parg->offset);
 828         }
 829
 830         perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
 831  out:
 832         preempt_enable();
 833 }
 834
 835 /* uprobe profile handler */
 836 static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs)
 837 {
 838         if (!uprobe_perf_filter(&tu->consumer, 0, current->mm))
 839                 return UPROBE_HANDLER_REMOVE;
 840
 841         if (!is_ret_probe(tu))
 842                 uprobe_perf_print(tu, 0, regs);
 843         return 0;
 844 }
 845
 846 static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func,
 847                                 struct pt_regs *regs)
 848 {
 849         uprobe_perf_print(tu, func, regs);
 850 }
 851 #endif  /* CONFIG_PERF_EVENTS */
 852
 853 static
 854 int trace_uprobe_register(struct ftrace_event_call *event, enum trace_reg type, void *data)
 855 {
 856         struct trace_uprobe *tu = event->data;
 857
 858         switch (type) {
 859         case TRACE_REG_REGISTER:
 860                 return probe_event_enable(tu, TP_FLAG_TRACE, NULL);
 861
 862         case TRACE_REG_UNREGISTER:
 863                 probe_event_disable(tu, TP_FLAG_TRACE);
 864                 return 0;
 865
 866 #ifdef CONFIG_PERF_EVENTS
 867         case TRACE_REG_PERF_REGISTER:
 868                 return probe_event_enable(tu, TP_FLAG_PROFILE, uprobe_perf_filter);
 869
 870         case TRACE_REG_PERF_UNREGISTER:
 871                 probe_event_disable(tu, TP_FLAG_PROFILE);
 872                 return 0;
 873
 874         case TRACE_REG_PERF_OPEN:
 875                 return uprobe_perf_open(tu, data);
 876
 877         case TRACE_REG_PERF_CLOSE:
 878                 return uprobe_perf_close(tu, data);
 879
 880 #endif
 881         default:
 882                 return 0;
 883         }
 884         return 0;
 885 }
 886
 887 static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
 888 {
 889         struct trace_uprobe *tu;
 890         int ret = 0;
 891
 892         tu = container_of(con, struct trace_uprobe, consumer);
 893         tu->nhit++;
 894
 895         if (tu->tp.flags & TP_FLAG_TRACE)
 896                 ret |= uprobe_trace_func(tu, regs);
 897
 898 #ifdef CONFIG_PERF_EVENTS
 899         if (tu->tp.flags & TP_FLAG_PROFILE)
 900                 ret |= uprobe_perf_func(tu, regs);
 901 #endif
 902         return ret;
 903 }
 904
 905 static int uretprobe_dispatcher(struct uprobe_consumer *con,
 906                                 unsigned long func, struct pt_regs *regs)
 907 {
 908         struct trace_uprobe *tu;
 909
 910         tu = container_of(con, struct trace_uprobe, consumer);
 911
 912         if (tu->tp.flags & TP_FLAG_TRACE)
 913                 uretprobe_trace_func(tu, func, regs);
 914
 915 #ifdef CONFIG_PERF_EVENTS
 916         if (tu->tp.flags & TP_FLAG_PROFILE)
 917                 uretprobe_perf_func(tu, func, regs);
 918 #endif
 919         return 0;
 920 }
 921
 922 static struct trace_event_functions uprobe_funcs = {
 923         .trace          = print_uprobe_event
 924 };
 925
 926 static int register_uprobe_event(struct trace_uprobe *tu)
 927 {
 928         struct ftrace_event_call *call = &tu->tp.call;
 929         int ret;
 930
 931         /* Initialize ftrace_event_call */
 932         INIT_LIST_HEAD(&call->class->fields);
 933         call->event.funcs = &uprobe_funcs;
 934         call->class->define_fields = uprobe_event_define_fields;
 935
 936         if (set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0)
 937                 return -ENOMEM;
 938
 939         ret = register_ftrace_event(&call->event);
 940         if (!ret) {
 941                 kfree(call->print_fmt);
 942                 return -ENODEV;
 943         }
 944         call->flags = 0;
 945         call->class->reg = trace_uprobe_register;
 946         call->data = tu;
 947         ret = trace_add_event_call(call);
 948
 949         if (ret) {
 950                 pr_info("Failed to register uprobe event: %s\n", call->name);
 951                 kfree(call->print_fmt);
 952                 unregister_ftrace_event(&call->event);
 953         }
 954
 955         return ret;
 956 }
 957
 958 static int unregister_uprobe_event(struct trace_uprobe *tu)
 959 {
 960         int ret;
 961
 962         /* tu->event is unregistered in trace_remove_event_call() */
 963         ret = trace_remove_event_call(&tu->tp.call);
 964         if (ret)
 965                 return ret;
 966         kfree(tu->tp.call.print_fmt);
 967         tu->tp.call.print_fmt = NULL;
 968         return 0;
 969 }
 970
 971 /* Make a trace interface for controling probe points */
 972 static __init int init_uprobe_trace(void)
 973 {
 974         struct dentry *d_tracer;
 975
 976         d_tracer = tracing_init_dentry();
 977         if (!d_tracer)
 978                 return 0;
 979
 980         trace_create_file("uprobe_events", 0644, d_tracer,
 981                                     NULL, &uprobe_events_ops);
 982         /* Profile interface */
 983         trace_create_file("uprobe_profile", 0444, d_tracer,
 984                                     NULL, &uprobe_profile_ops);
 985         return 0;
 986 }
 987
 988 fs_initcall(init_uprobe_trace);