[linux-block.git] / include / linux / trace_recursion.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_TRACE_RECURSION_H
#define _LINUX_TRACE_RECURSION_H

#include <linux/interrupt.h>
#include <linux/sched.h>

#ifdef CONFIG_TRACING

/* Only current can touch trace_recursion */

/*
 * For function tracing recursion:
 *  The order of these bits are important.
 *
 *  When function tracing occurs, the following steps are made:
 *   If arch does not support a ftrace feature:
 *    call internal function (uses INTERNAL bits) which calls...
 *   The function callback, which can use the FTRACE bits to
 *    check for recursion.
 */
enum {
	/* Function recursion bits */
	TRACE_FTRACE_BIT,
	TRACE_FTRACE_NMI_BIT,
	TRACE_FTRACE_IRQ_BIT,
	TRACE_FTRACE_SIRQ_BIT,
	TRACE_FTRACE_TRANSITION_BIT,

	/* Internal use recursion bits */
	TRACE_INTERNAL_BIT,
	TRACE_INTERNAL_NMI_BIT,
	TRACE_INTERNAL_IRQ_BIT,
	TRACE_INTERNAL_SIRQ_BIT,
	TRACE_INTERNAL_TRANSITION_BIT,

	TRACE_BRANCH_BIT,
/*
 * Abuse of the trace_recursion.
 * As we need a way to maintain state if we are tracing the function
 * graph in irq because we want to trace a particular function that
 * was called in irq context but we have irq tracing off. Since this
 * can only be modified by current, we can reuse trace_recursion.
 */
	TRACE_IRQ_BIT,

	/* Set if the function is in the set_graph_function file */
	TRACE_GRAPH_BIT,

	/*
	 * In the very unlikely case that an interrupt came in
	 * at a start of graph tracing, and we want to trace
	 * the function in that interrupt, the depth can be greater
	 * than zero, because of the preempted start of a previous
	 * trace. In an even more unlikely case, depth could be 2
	 * if a softirq interrupted the start of graph tracing,
	 * followed by an interrupt preempting a start of graph
	 * tracing in the softirq, and depth can even be 3
	 * if an NMI came in at the start of an interrupt function
	 * that preempted a softirq start of a function that
	 * preempted normal context!!!! Luckily, it can't be
	 * greater than 3, so the next two bits are a mask
	 * of what the depth is when we set TRACE_GRAPH_BIT
	 */

	TRACE_GRAPH_DEPTH_START_BIT,
	TRACE_GRAPH_DEPTH_END_BIT,

	/*
	 * To implement set_graph_notrace, if this bit is set, we ignore
	 * function graph tracing of called functions, until the return
	 * function is called to clear it.
	 */
	TRACE_GRAPH_NOTRACE_BIT,

	/* Used to prevent recursion recording from recursing. */
	TRACE_RECORD_RECURSION_BIT,
};

#define trace_recursion_set(bit)	do { (current)->trace_recursion |= (1<<(bit)); } while (0)
#define trace_recursion_clear(bit)	do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
#define trace_recursion_test(bit)	((current)->trace_recursion & (1<<(bit)))

#define trace_recursion_depth() \
	(((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3)
#define trace_recursion_set_depth(depth) \
	do {								\
		current->trace_recursion &=				\
			~(3 << TRACE_GRAPH_DEPTH_START_BIT);		\
		current->trace_recursion |=				\
			((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT;	\
	} while (0)

#define TRACE_CONTEXT_BITS	4

#define TRACE_FTRACE_START	TRACE_FTRACE_BIT

#define TRACE_LIST_START	TRACE_INTERNAL_BIT

#define TRACE_CONTEXT_MASK	((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)

/*
 * Used for setting context
 *  NMI     = 0
 *  IRQ     = 1
 *  SOFTIRQ = 2
 *  NORMAL  = 3
 */
enum {
	TRACE_CTX_NMI,
	TRACE_CTX_IRQ,
	TRACE_CTX_SOFTIRQ,
	TRACE_CTX_NORMAL,
	TRACE_CTX_TRANSITION,
};

static __always_inline int trace_get_context_bit(void)
{
	unsigned char bit = interrupt_context_level();

	return TRACE_CTX_NORMAL - bit;
}

#ifdef CONFIG_FTRACE_RECORD_RECURSION
extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip);
# define do_ftrace_record_recursion(ip, pip)				\
	do {								\
		if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
			trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
			ftrace_record_recursion(ip, pip);		\
			trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
		}							\
	} while (0)
#else
# define do_ftrace_record_recursion(ip, pip)	do { } while (0)
#endif

#ifdef CONFIG_ARCH_WANTS_NO_INSTR
# define trace_warn_on_no_rcu(ip)					\
	({								\
		bool __ret = !rcu_is_watching();			\
		if (__ret && !trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
			trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
			WARN_ONCE(true, "RCU not on for: %pS\n", (void *)ip); \
			trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
		}							\
		__ret;							\
	})
#else
# define trace_warn_on_no_rcu(ip)	false
#endif

/*
 * Preemption is promised to be disabled when return bit >= 0.
 */
static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip,
							int start)
{
	unsigned int val = READ_ONCE(current->trace_recursion);
	int bit;

	if (trace_warn_on_no_rcu(ip))
		return -1;

	bit = trace_get_context_bit() + start;
	if (unlikely(val & (1 << bit))) {
		/*
		 * If an interrupt occurs during a trace, and another trace
		 * happens in that interrupt but before the preempt_count is
		 * updated to reflect the new interrupt context, then this
		 * will think a recursion occurred, and the event will be dropped.
		 * Let a single instance happen via the TRANSITION_BIT to
		 * not drop those events.
		 */
		bit = TRACE_CTX_TRANSITION + start;
		if (val & (1 << bit)) {
			do_ftrace_record_recursion(ip, pip);
			return -1;
		}
	}

	val |= 1 << bit;
	current->trace_recursion = val;
	barrier();

	preempt_disable_notrace();

	return bit;
}

/*
 * Preemption will be enabled (if it was previously enabled).
 */
static __always_inline void trace_clear_recursion(int bit)
{
	preempt_enable_notrace();
	barrier();
	trace_recursion_clear(bit);
}

/**
 * ftrace_test_recursion_trylock - tests for recursion in same context
 *
 * Use this for ftrace callbacks. This will detect if the function
 * tracing recursed in the same context (normal vs interrupt),
 *
 * Returns: -1 if a recursion happened.
 *           >= 0 if no recursion.
 */
static __always_inline int ftrace_test_recursion_trylock(unsigned long ip,
							 unsigned long parent_ip)
{
	return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START);
}

/**
 * ftrace_test_recursion_unlock - called when function callback is complete
 * @bit: The return of a successful ftrace_test_recursion_trylock()
 *
 * This is used at the end of a ftrace callback.
 */
static __always_inline void ftrace_test_recursion_unlock(int bit)
{
	trace_clear_recursion(bit);
}

#endif /* CONFIG_TRACING */
#endif /* _LINUX_TRACE_RECURSION_H */
Commit	Line	Data
0264c8c9 SRV	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _LINUX_TRACE_RECURSION_H
	3	#define _LINUX_TRACE_RECURSION_H
	4
	5	#include <linux/interrupt.h>
	6	#include <linux/sched.h>
	7
	8	#ifdef CONFIG_TRACING
	9
	10	/* Only current can touch trace_recursion */
	11
	12	/*
	13	* For function tracing recursion:
	14	* The order of these bits are important.
	15	*
	16	* When function tracing occurs, the following steps are made:
	17	* If arch does not support a ftrace feature:
	18	* call internal function (uses INTERNAL bits) which calls...
0264c8c9 SRV	19	* The function callback, which can use the FTRACE bits to
0264c8c9 SRV	20	* check for recursion.
0264c8c9 SRV	21	*/
	22	enum {
	23	/* Function recursion bits */
	24	TRACE_FTRACE_BIT,
	25	TRACE_FTRACE_NMI_BIT,
	26	TRACE_FTRACE_IRQ_BIT,
	27	TRACE_FTRACE_SIRQ_BIT,
ed65df63	28	TRACE_FTRACE_TRANSITION_BIT,
0264c8c9	29
ed65df63	30	/* Internal use recursion bits */
0264c8c9 SRV	31	TRACE_INTERNAL_BIT,
	32	TRACE_INTERNAL_NMI_BIT,
	33	TRACE_INTERNAL_IRQ_BIT,
	34	TRACE_INTERNAL_SIRQ_BIT,
ed65df63	35	TRACE_INTERNAL_TRANSITION_BIT,
0264c8c9 SRV	36
	37	TRACE_BRANCH_BIT,
	38	/*
	39	* Abuse of the trace_recursion.
	40	* As we need a way to maintain state if we are tracing the function
	41	* graph in irq because we want to trace a particular function that
	42	* was called in irq context but we have irq tracing off. Since this
	43	* can only be modified by current, we can reuse trace_recursion.
	44	*/
	45	TRACE_IRQ_BIT,
	46
	47	/* Set if the function is in the set_graph_function file */
	48	TRACE_GRAPH_BIT,
	49
	50	/*
	51	* In the very unlikely case that an interrupt came in
	52	* at a start of graph tracing, and we want to trace
	53	* the function in that interrupt, the depth can be greater
	54	* than zero, because of the preempted start of a previous
	55	* trace. In an even more unlikely case, depth could be 2
	56	* if a softirq interrupted the start of graph tracing,
	57	* followed by an interrupt preempting a start of graph
	58	* tracing in the softirq, and depth can even be 3
	59	* if an NMI came in at the start of an interrupt function
	60	* that preempted a softirq start of a function that
	61	* preempted normal context!!!! Luckily, it can't be
	62	* greater than 3, so the next two bits are a mask
	63	* of what the depth is when we set TRACE_GRAPH_BIT
	64	*/
	65
	66	TRACE_GRAPH_DEPTH_START_BIT,
	67	TRACE_GRAPH_DEPTH_END_BIT,
	68
	69	/*
	70	* To implement set_graph_notrace, if this bit is set, we ignore
	71	* function graph tracing of called functions, until the return
	72	* function is called to clear it.
	73	*/
	74	TRACE_GRAPH_NOTRACE_BIT,
	75
773c1670 SRV	76	/* Used to prevent recursion recording from recursing. */
773c1670 SRV	77	TRACE_RECORD_RECURSION_BIT,
0264c8c9 SRV	78	};
	79
	80	#define trace_recursion_set(bit) do { (current)->trace_recursion \|= (1<<(bit)); } while (0)
	81	#define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
	82	#define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit)))
	83
	84	#define trace_recursion_depth() \
	85	(((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3)
	86	#define trace_recursion_set_depth(depth) \
	87	do { \
	88	current->trace_recursion &= \
	89	~(3 << TRACE_GRAPH_DEPTH_START_BIT); \
	90	current->trace_recursion \|= \
	91	((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT; \
	92	} while (0)
	93
	94	#define TRACE_CONTEXT_BITS 4
	95
	96	#define TRACE_FTRACE_START TRACE_FTRACE_BIT
0264c8c9 SRV	97
0264c8c9 SRV	98	#define TRACE_LIST_START TRACE_INTERNAL_BIT
0264c8c9	99
ed65df63	100	#define TRACE_CONTEXT_MASK ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
0264c8c9	101
da5afbeb SRV	102	/*
	103	* Used for setting context
	104	* NMI = 0
	105	* IRQ = 1
	106	* SOFTIRQ = 2
	107	* NORMAL = 3
	108	*/
	109	enum {
	110	TRACE_CTX_NMI,
	111	TRACE_CTX_IRQ,
	112	TRACE_CTX_SOFTIRQ,
	113	TRACE_CTX_NORMAL,
ed65df63	114	TRACE_CTX_TRANSITION,
da5afbeb SRV	115	};
da5afbeb SRV	116
0264c8c9 SRV	117	static __always_inline int trace_get_context_bit(void)
0264c8c9 SRV	118	{
91ebe8bc	119	unsigned char bit = interrupt_context_level();
0264c8c9	120
9b84fadc	121	return TRACE_CTX_NORMAL - bit;
0264c8c9 SRV	122	}
0264c8c9 SRV	123
773c1670 SRV	124	#ifdef CONFIG_FTRACE_RECORD_RECURSION
	125	extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip);
	126	# define do_ftrace_record_recursion(ip, pip) \
	127	do { \
	128	if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
	129	trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
	130	ftrace_record_recursion(ip, pip); \
	131	trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
	132	} \
	133	} while (0)
	134	#else
	135	# define do_ftrace_record_recursion(ip, pip) do { } while (0)
	136	#endif
	137
d099dbfd PZ	138	#ifdef CONFIG_ARCH_WANTS_NO_INSTR
	139	# define trace_warn_on_no_rcu(ip) \
	140	({ \
	141	bool __ret = !rcu_is_watching(); \
	142	if (__ret && !trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
	143	trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
	144	WARN_ONCE(true, "RCU not on for: %pS\n", (void *)ip); \
	145	trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
	146	} \
	147	__ret; \
	148	})
	149	#else
	150	# define trace_warn_on_no_rcu(ip) false
	151	#endif
	152
ce5e4803	153	/*
	154	* Preemption is promised to be disabled when return bit >= 0.
	155	*/
773c1670	156	static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip,
ed65df63	157	int start)
0264c8c9	158	{
7b68621f	159	unsigned int val = READ_ONCE(current->trace_recursion);
0264c8c9 SRV	160	int bit;
0264c8c9 SRV	161
d099dbfd PZ	162	if (trace_warn_on_no_rcu(ip))
	163	return -1;
	164
0264c8c9 SRV	165	bit = trace_get_context_bit() + start;
	166	if (unlikely(val & (1 << bit))) {
	167	/*
bce5c81c SRV	168	* If an interrupt occurs during a trace, and another trace
	169	* happens in that interrupt but before the preempt_count is
	170	* updated to reflect the new interrupt context, then this
	171	* will think a recursion occurred, and the event will be dropped.
	172	* Let a single instance happen via the TRANSITION_BIT to
	173	* not drop those events.
0264c8c9	174	*/
ed65df63	175	bit = TRACE_CTX_TRANSITION + start;
7b68621f	176	if (val & (1 << bit)) {
773c1670	177	do_ftrace_record_recursion(ip, pip);
0264c8c9	178	return -1;
773c1670	179	}
0264c8c9 SRV	180	}
0264c8c9 SRV	181
0264c8c9 SRV	182	val \|= 1 << bit;
	183	current->trace_recursion = val;
	184	barrier();
	185
ce5e4803	186	preempt_disable_notrace();
ce5e4803	187
ed65df63	188	return bit;
0264c8c9 SRV	189	}
0264c8c9 SRV	190
ce5e4803	191	/*
	192	* Preemption will be enabled (if it was previously enabled).
	193	*/
0264c8c9 SRV	194	static __always_inline void trace_clear_recursion(int bit)
0264c8c9 SRV	195	{
ce5e4803	196	preempt_enable_notrace();
0264c8c9	197	barrier();
7b68621f	198	trace_recursion_clear(bit);
0264c8c9 SRV	199	}
0264c8c9 SRV	200
6e4eb9cb SRV	201	/**
	202	* ftrace_test_recursion_trylock - tests for recursion in same context
	203	*
	204	* Use this for ftrace callbacks. This will detect if the function
	205	* tracing recursed in the same context (normal vs interrupt),
	206	*
	207	* Returns: -1 if a recursion happened.
ce5e4803	208	* >= 0 if no recursion.
6e4eb9cb	209	*/
773c1670 SRV	210	static __always_inline int ftrace_test_recursion_trylock(unsigned long ip,
773c1670 SRV	211	unsigned long parent_ip)
6e4eb9cb	212	{
ed65df63	213	return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START);
6e4eb9cb SRV	214	}
	215
	216	/**
	217	* ftrace_test_recursion_unlock - called when function callback is complete
	218	* @bit: The return of a successful ftrace_test_recursion_trylock()
	219	*
	220	* This is used at the end of a ftrace callback.
	221	*/
	222	static __always_inline void ftrace_test_recursion_unlock(int bit)
	223	{
	224	trace_clear_recursion(bit);
	225	}
	226
0264c8c9 SRV	227	#endif /* CONFIG_TRACING */
0264c8c9 SRV	228	#endif /* _LINUX_TRACE_RECURSION_H */