Merge tag 'trace-v6.3-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/trace...

[linux-block.git] / kernel / trace / ftrace.c

diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index a47f7d93e32d24d9cc61fdccf60eb9722a7f38b7..9b2803c7a18f8e54fe99b02bd22794d0e52f58d8 100644 (file)
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -125,6 +125,33 @@ struct ftrace_ops global_ops;
 void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
                          struct ftrace_ops *op, struct ftrace_regs *fregs);
 
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
+/*
+ * Stub used to invoke the list ops without requiring a separate trampoline.
+ */
+const struct ftrace_ops ftrace_list_ops = {
+       .func   = ftrace_ops_list_func,
+       .flags  = FTRACE_OPS_FL_STUB,
+};
+
+static void ftrace_ops_nop_func(unsigned long ip, unsigned long parent_ip,
+                               struct ftrace_ops *op,
+                               struct ftrace_regs *fregs)
+{
+       /* do nothing */
+}
+
+/*
+ * Stub used when a call site is disabled. May be called transiently by threads
+ * which have made it into ftrace_caller but haven't yet recovered the ops at
+ * the point the call site is disabled.
+ */
+const struct ftrace_ops ftrace_nop_ops = {
+       .func   = ftrace_ops_nop_func,
+       .flags  = FTRACE_OPS_FL_STUB,
+};
+#endif
+
 static inline void ftrace_ops_init(struct ftrace_ops *ops)
 {
 #ifdef CONFIG_DYNAMIC_FTRACE
@@ -1820,6 +1847,18 @@ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
                         * if rec count is zero.
                         */
                }
+
+               /*
+                * If the rec has a single associated ops, and ops->func can be
+                * called directly, allow the call site to call via the ops.
+                */
+               if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS) &&
+                   ftrace_rec_count(rec) == 1 &&
+                   ftrace_ops_get_func(ops) == ops->func)
+                       rec->flags |= FTRACE_FL_CALL_OPS;
+               else
+                       rec->flags &= ~FTRACE_FL_CALL_OPS;
+
                count++;
 
                /* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
@@ -2114,8 +2153,9 @@ void ftrace_bug(int failed, struct dyn_ftrace *rec)
                struct ftrace_ops *ops = NULL;
 
                pr_info("ftrace record flags: %lx\n", rec->flags);
-               pr_cont(" (%ld)%s", ftrace_rec_count(rec),
-                       rec->flags & FTRACE_FL_REGS ? " R" : "  ");
+               pr_cont(" (%ld)%s%s", ftrace_rec_count(rec),
+                       rec->flags & FTRACE_FL_REGS ? " R" : "  ",
+                       rec->flags & FTRACE_FL_CALL_OPS ? " O" : "  ");
                if (rec->flags & FTRACE_FL_TRAMP_EN) {
                        ops = ftrace_find_tramp_ops_any(rec);
                        if (ops) {
@@ -2183,6 +2223,7 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update)
                 * want the direct enabled (it will be done via the
                 * direct helper). But if DIRECT_EN is set, and
                 * the count is not one, we need to clear it.
+                *
                 */
                if (ftrace_rec_count(rec) == 1) {
                        if (!(rec->flags & FTRACE_FL_DIRECT) !=
@@ -2191,6 +2232,19 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update)
                } else if (rec->flags & FTRACE_FL_DIRECT_EN) {
                        flag |= FTRACE_FL_DIRECT;
                }
+
+               /*
+                * Ops calls are special, as count matters.
+                * As with direct calls, they must only be enabled when count
+                * is one, otherwise they'll be handled via the list ops.
+                */
+               if (ftrace_rec_count(rec) == 1) {
+                       if (!(rec->flags & FTRACE_FL_CALL_OPS) !=
+                           !(rec->flags & FTRACE_FL_CALL_OPS_EN))
+                               flag |= FTRACE_FL_CALL_OPS;
+               } else if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
+                       flag |= FTRACE_FL_CALL_OPS;
+               }
        }
 
        /* If the state of this record hasn't changed, then do nothing */
@@ -2235,6 +2289,21 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update)
                                        rec->flags &= ~FTRACE_FL_DIRECT_EN;
                                }
                        }
+
+                       if (flag & FTRACE_FL_CALL_OPS) {
+                               if (ftrace_rec_count(rec) == 1) {
+                                       if (rec->flags & FTRACE_FL_CALL_OPS)
+                                               rec->flags |= FTRACE_FL_CALL_OPS_EN;
+                                       else
+                                               rec->flags &= ~FTRACE_FL_CALL_OPS_EN;
+                               } else {
+                                       /*
+                                        * Can only call directly if there's
+                                        * only one set of associated ops.
+                                        */
+                                       rec->flags &= ~FTRACE_FL_CALL_OPS_EN;
+                               }
+                       }
                }
 
                /*
@@ -2264,7 +2333,8 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update)
                         * and REGS states. The _EN flags must be disabled though.
                         */
                        rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
-                                       FTRACE_FL_REGS_EN | FTRACE_FL_DIRECT_EN);
+                                       FTRACE_FL_REGS_EN | FTRACE_FL_DIRECT_EN |
+                                       FTRACE_FL_CALL_OPS_EN);
        }
 
        ftrace_bug_type = FTRACE_BUG_NOP;
@@ -2437,6 +2507,25 @@ ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
        return NULL;
 }
 
+struct ftrace_ops *
+ftrace_find_unique_ops(struct dyn_ftrace *rec)
+{
+       struct ftrace_ops *op, *found = NULL;
+       unsigned long ip = rec->ip;
+
+       do_for_each_ftrace_op(op, ftrace_ops_list) {
+
+               if (hash_contains_ip(ip, op->func_hash)) {
+                       if (found)
+                               return NULL;
+                       found = op;
+               }
+
+       } while_for_each_ftrace_op(op);
+
+       return found;
+}
+
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
 /* Protected by rcu_tasks for reading, and direct_mutex for writing */
 static struct ftrace_hash *direct_functions = EMPTY_HASH;
@@ -3786,11 +3875,12 @@ static int t_show(struct seq_file *m, void *v)
        if (iter->flags & FTRACE_ITER_ENABLED) {
                struct ftrace_ops *ops;
 
-               seq_printf(m, " (%ld)%s%s%s",
+               seq_printf(m, " (%ld)%s%s%s%s",
                           ftrace_rec_count(rec),
                           rec->flags & FTRACE_FL_REGS ? " R" : "  ",
                           rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ",
-                          rec->flags & FTRACE_FL_DIRECT ? " D" : "  ");
+                          rec->flags & FTRACE_FL_DIRECT ? " D" : "  ",
+                          rec->flags & FTRACE_FL_CALL_OPS ? " O" : "  ");
                if (rec->flags & FTRACE_FL_TRAMP_EN) {
                        ops = ftrace_find_tramp_ops_any(rec);
                        if (ops) {
@@ -3806,6 +3896,15 @@ static int t_show(struct seq_file *m, void *v)
                } else {
                        add_trampoline_func(m, NULL, rec);
                }
+               if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
+                       ops = ftrace_find_unique_ops(rec);
+                       if (ops) {
+                               seq_printf(m, "\tops: %pS (%pS)",
+                                          ops, ops->func);
+                       } else {
+                               seq_puts(m, "\tops: ERROR!");
+                       }
+               }
                if (rec->flags & FTRACE_FL_DIRECT) {
                        unsigned long direct;
 
@@ -8346,7 +8445,7 @@ int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt, unsigned long *a
        found_all = kallsyms_on_each_symbol(kallsyms_callback, &args);
        if (found_all)
                return 0;
-       found_all = module_kallsyms_on_each_symbol(kallsyms_callback, &args);
+       found_all = module_kallsyms_on_each_symbol(NULL, kallsyms_callback, &args);
        return found_all ? 0 : -ESRCH;
 }