[linux-block.git] / include / linux / trace_recursion.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_TRACE_RECURSION_H
#define _LINUX_TRACE_RECURSION_H

#include <linux/interrupt.h>
#include <linux/sched.h>

#ifdef CONFIG_TRACING

/* Only current can touch trace_recursion */

/*
 * For function tracing recursion:
 *  The order of these bits are important.
 *
 *  When function tracing occurs, the following steps are made:
 *   If arch does not support a ftrace feature:
 *    call internal function (uses INTERNAL bits) which calls...
 *   If callback is registered to the "global" list, the list
 *    function is called and recursion checks the GLOBAL bits.
 *    then this function calls...
 *   The function callback, which can use the FTRACE bits to
 *    check for recursion.
 *
 * Now if the arch does not support a feature, and it calls
 * the global list function which calls the ftrace callback
 * all three of these steps will do a recursion protection.
 * There's no reason to do one if the previous caller already
 * did. The recursion that we are protecting against will
 * go through the same steps again.
 *
 * To prevent the multiple recursion checks, if a recursion
 * bit is set that is higher than the MAX bit of the current
 * check, then we know that the check was made by the previous
 * caller, and we can skip the current check.
 */
enum {
	/* Function recursion bits */
	TRACE_FTRACE_BIT,
	TRACE_FTRACE_NMI_BIT,
	TRACE_FTRACE_IRQ_BIT,
	TRACE_FTRACE_SIRQ_BIT,

	/* INTERNAL_BITs must be greater than FTRACE_BITs */
	TRACE_INTERNAL_BIT,
	TRACE_INTERNAL_NMI_BIT,
	TRACE_INTERNAL_IRQ_BIT,
	TRACE_INTERNAL_SIRQ_BIT,

	TRACE_BRANCH_BIT,
/*
 * Abuse of the trace_recursion.
 * As we need a way to maintain state if we are tracing the function
 * graph in irq because we want to trace a particular function that
 * was called in irq context but we have irq tracing off. Since this
 * can only be modified by current, we can reuse trace_recursion.
 */
	TRACE_IRQ_BIT,

	/* Set if the function is in the set_graph_function file */
	TRACE_GRAPH_BIT,

	/*
	 * In the very unlikely case that an interrupt came in
	 * at a start of graph tracing, and we want to trace
	 * the function in that interrupt, the depth can be greater
	 * than zero, because of the preempted start of a previous
	 * trace. In an even more unlikely case, depth could be 2
	 * if a softirq interrupted the start of graph tracing,
	 * followed by an interrupt preempting a start of graph
	 * tracing in the softirq, and depth can even be 3
	 * if an NMI came in at the start of an interrupt function
	 * that preempted a softirq start of a function that
	 * preempted normal context!!!! Luckily, it can't be
	 * greater than 3, so the next two bits are a mask
	 * of what the depth is when we set TRACE_GRAPH_BIT
	 */

	TRACE_GRAPH_DEPTH_START_BIT,
	TRACE_GRAPH_DEPTH_END_BIT,

	/*
	 * To implement set_graph_notrace, if this bit is set, we ignore
	 * function graph tracing of called functions, until the return
	 * function is called to clear it.
	 */
	TRACE_GRAPH_NOTRACE_BIT,

	/*
	 * When transitioning between context, the preempt_count() may
	 * not be correct. Allow for a single recursion to cover this case.
	 */
	TRACE_TRANSITION_BIT,

	/* Used to prevent recursion recording from recursing. */
	TRACE_RECORD_RECURSION_BIT,
};

#define trace_recursion_set(bit)	do { (current)->trace_recursion |= (1<<(bit)); } while (0)
#define trace_recursion_clear(bit)	do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
#define trace_recursion_test(bit)	((current)->trace_recursion & (1<<(bit)))

#define trace_recursion_depth() \
	(((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3)
#define trace_recursion_set_depth(depth) \
	do {								\
		current->trace_recursion &=				\
			~(3 << TRACE_GRAPH_DEPTH_START_BIT);		\
		current->trace_recursion |=				\
			((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT;	\
	} while (0)

#define TRACE_CONTEXT_BITS	4

#define TRACE_FTRACE_START	TRACE_FTRACE_BIT
#define TRACE_FTRACE_MAX	((1 << (TRACE_FTRACE_START + TRACE_CONTEXT_BITS)) - 1)

#define TRACE_LIST_START	TRACE_INTERNAL_BIT
#define TRACE_LIST_MAX		((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)

#define TRACE_CONTEXT_MASK	TRACE_LIST_MAX

/*
 * Used for setting context
 *  NMI     = 0
 *  IRQ     = 1
 *  SOFTIRQ = 2
 *  NORMAL  = 3
 */
enum {
	TRACE_CTX_NMI,
	TRACE_CTX_IRQ,
	TRACE_CTX_SOFTIRQ,
	TRACE_CTX_NORMAL,
};

static __always_inline int trace_get_context_bit(void)
{
	unsigned long pc = preempt_count();

	if (!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)))
		return TRACE_CTX_NORMAL;
	else
		return pc & NMI_MASK ? TRACE_CTX_NMI :
			pc & HARDIRQ_MASK ? TRACE_CTX_IRQ : TRACE_CTX_SOFTIRQ;
}

#ifdef CONFIG_FTRACE_RECORD_RECURSION
extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip);
# define do_ftrace_record_recursion(ip, pip)				\
	do {								\
		if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
			trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
			ftrace_record_recursion(ip, pip);		\
			trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
		}							\
	} while (0)
#else
# define do_ftrace_record_recursion(ip, pip)	do { } while (0)
#endif

static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip,
							int start, int max)
{
	unsigned int val = READ_ONCE(current->trace_recursion);
	int bit;

	/* A previous recursion check was made */
	if ((val & TRACE_CONTEXT_MASK) > max)
		return 0;

	bit = trace_get_context_bit() + start;
	if (unlikely(val & (1 << bit))) {
		/*
		 * It could be that preempt_count has not been updated during
		 * a switch between contexts. Allow for a single recursion.
		 */
		bit = TRACE_TRANSITION_BIT;
		if (val & (1 << bit)) {
			do_ftrace_record_recursion(ip, pip);
			return -1;
		}
	} else {
		/* Normal check passed, clear the transition to allow it again */
		val &= ~(1 << TRACE_TRANSITION_BIT);
	}

	val |= 1 << bit;
	current->trace_recursion = val;
	barrier();

	return bit + 1;
}

static __always_inline void trace_clear_recursion(int bit)
{
	if (!bit)
		return;

	barrier();
	bit--;
	trace_recursion_clear(bit);
}

/**
 * ftrace_test_recursion_trylock - tests for recursion in same context
 *
 * Use this for ftrace callbacks. This will detect if the function
 * tracing recursed in the same context (normal vs interrupt),
 *
 * Returns: -1 if a recursion happened.
 *           >= 0 if no recursion
 */
static __always_inline int ftrace_test_recursion_trylock(unsigned long ip,
							 unsigned long parent_ip)
{
	return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START, TRACE_FTRACE_MAX);
}

/**
 * ftrace_test_recursion_unlock - called when function callback is complete
 * @bit: The return of a successful ftrace_test_recursion_trylock()
 *
 * This is used at the end of a ftrace callback.
 */
static __always_inline void ftrace_test_recursion_unlock(int bit)
{
	trace_clear_recursion(bit);
}

#endif /* CONFIG_TRACING */
#endif /* _LINUX_TRACE_RECURSION_H */
Commit	Line	Data
0264c8c9 SRV	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _LINUX_TRACE_RECURSION_H
	3	#define _LINUX_TRACE_RECURSION_H
	4
	5	#include <linux/interrupt.h>
	6	#include <linux/sched.h>
	7
	8	#ifdef CONFIG_TRACING
	9
	10	/* Only current can touch trace_recursion */
	11
	12	/*
	13	* For function tracing recursion:
	14	* The order of these bits are important.
	15	*
	16	* When function tracing occurs, the following steps are made:
	17	* If arch does not support a ftrace feature:
	18	* call internal function (uses INTERNAL bits) which calls...
	19	* If callback is registered to the "global" list, the list
	20	* function is called and recursion checks the GLOBAL bits.
	21	* then this function calls...
	22	* The function callback, which can use the FTRACE bits to
	23	* check for recursion.
	24	*
	25	* Now if the arch does not support a feature, and it calls
	26	* the global list function which calls the ftrace callback
	27	* all three of these steps will do a recursion protection.
	28	* There's no reason to do one if the previous caller already
	29	* did. The recursion that we are protecting against will
	30	* go through the same steps again.
	31	*
	32	* To prevent the multiple recursion checks, if a recursion
	33	* bit is set that is higher than the MAX bit of the current
	34	* check, then we know that the check was made by the previous
	35	* caller, and we can skip the current check.
	36	*/
	37	enum {
	38	/* Function recursion bits */
	39	TRACE_FTRACE_BIT,
	40	TRACE_FTRACE_NMI_BIT,
	41	TRACE_FTRACE_IRQ_BIT,
	42	TRACE_FTRACE_SIRQ_BIT,
	43
	44	/* INTERNAL_BITs must be greater than FTRACE_BITs */
	45	TRACE_INTERNAL_BIT,
	46	TRACE_INTERNAL_NMI_BIT,
	47	TRACE_INTERNAL_IRQ_BIT,
	48	TRACE_INTERNAL_SIRQ_BIT,
	49
	50	TRACE_BRANCH_BIT,
	51	/*
	52	* Abuse of the trace_recursion.
	53	* As we need a way to maintain state if we are tracing the function
	54	* graph in irq because we want to trace a particular function that
	55	* was called in irq context but we have irq tracing off. Since this
	56	* can only be modified by current, we can reuse trace_recursion.
	57	*/
	58	TRACE_IRQ_BIT,
	59
	60	/* Set if the function is in the set_graph_function file */
	61	TRACE_GRAPH_BIT,
	62
	63	/*
	64	* In the very unlikely case that an interrupt came in
65	* at a start of graph tracing, and we want to trace
66	* the function in that interrupt, the depth can be greater
67	* than zero, because of the preempted start of a previous
68	* trace. In an even more unlikely case, depth could be 2
69	* if a softirq interrupted the start of graph tracing,
70	* followed by an interrupt preempting a start of graph
71	* tracing in the softirq, and depth can even be 3
72	* if an NMI came in at the start of an interrupt function
73	* that preempted a softirq start of a function that
74	* preempted normal context!!!! Luckily, it can't be
75	* greater than 3, so the next two bits are a mask
76	* of what the depth is when we set TRACE_GRAPH_BIT
77	*/
78
79	TRACE_GRAPH_DEPTH_START_BIT,
80	TRACE_GRAPH_DEPTH_END_BIT,
81
82	/*
83	* To implement set_graph_notrace, if this bit is set, we ignore
84	* function graph tracing of called functions, until the return
85	* function is called to clear it.
86	*/
87	TRACE_GRAPH_NOTRACE_BIT,
88
89	/*
90	* When transitioning between context, the preempt_count() may
91	* not be correct. Allow for a single recursion to cover this case.
92	*/
93	TRACE_TRANSITION_BIT,
773c1670 SRV	94
	95	/* Used to prevent recursion recording from recursing. */
	96	TRACE_RECORD_RECURSION_BIT,
0264c8c9 SRV	97	};
	98
	99	#define trace_recursion_set(bit) do { (current)->trace_recursion \|= (1<<(bit)); } while (0)
	100	#define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
	101	#define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit)))
	102
	103	#define trace_recursion_depth() \
	104	(((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3)
	105	#define trace_recursion_set_depth(depth) \
	106	do { \
	107	current->trace_recursion &= \
	108	~(3 << TRACE_GRAPH_DEPTH_START_BIT); \
	109	current->trace_recursion \|= \
	110	((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT; \
	111	} while (0)
	112
	113	#define TRACE_CONTEXT_BITS 4
	114
	115	#define TRACE_FTRACE_START TRACE_FTRACE_BIT
	116	#define TRACE_FTRACE_MAX ((1 << (TRACE_FTRACE_START + TRACE_CONTEXT_BITS)) - 1)
	117
	118	#define TRACE_LIST_START TRACE_INTERNAL_BIT
	119	#define TRACE_LIST_MAX ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
	120
	121	#define TRACE_CONTEXT_MASK TRACE_LIST_MAX
	122
da5afbeb SRV	123	/*
	124	* Used for setting context
	125	* NMI = 0
	126	* IRQ = 1
	127	* SOFTIRQ = 2
	128	* NORMAL = 3
	129	*/
	130	enum {
	131	TRACE_CTX_NMI,
	132	TRACE_CTX_IRQ,
	133	TRACE_CTX_SOFTIRQ,
	134	TRACE_CTX_NORMAL,
	135	};
	136
0264c8c9 SRV	137	static __always_inline int trace_get_context_bit(void)
0264c8c9 SRV	138	{
da5afbeb	139	unsigned long pc = preempt_count();
0264c8c9	140
da5afbeb SRV	141	if (!(pc & (NMI_MASK \| HARDIRQ_MASK \| SOFTIRQ_OFFSET)))
	142	return TRACE_CTX_NORMAL;
	143	else
	144	return pc & NMI_MASK ? TRACE_CTX_NMI :
	145	pc & HARDIRQ_MASK ? TRACE_CTX_IRQ : TRACE_CTX_SOFTIRQ;
0264c8c9 SRV	146	}
0264c8c9 SRV	147
773c1670 SRV	148	#ifdef CONFIG_FTRACE_RECORD_RECURSION
	149	extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip);
	150	# define do_ftrace_record_recursion(ip, pip) \
	151	do { \
	152	if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
	153	trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
	154	ftrace_record_recursion(ip, pip); \
	155	trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
	156	} \
	157	} while (0)
	158	#else
	159	# define do_ftrace_record_recursion(ip, pip) do { } while (0)
	160	#endif
	161
	162	static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip,
	163	int start, int max)
0264c8c9	164	{
7b68621f	165	unsigned int val = READ_ONCE(current->trace_recursion);
0264c8c9 SRV	166	int bit;
	167
	168	/* A previous recursion check was made */
	169	if ((val & TRACE_CONTEXT_MASK) > max)
	170	return 0;
	171
	172	bit = trace_get_context_bit() + start;
	173	if (unlikely(val & (1 << bit))) {
	174	/*
	175	* It could be that preempt_count has not been updated during
	176	* a switch between contexts. Allow for a single recursion.
	177	*/
	178	bit = TRACE_TRANSITION_BIT;
7b68621f	179	if (val & (1 << bit)) {
773c1670	180	do_ftrace_record_recursion(ip, pip);
0264c8c9	181	return -1;
773c1670	182	}
7b68621f SRV	183	} else {
	184	/* Normal check passed, clear the transition to allow it again */
	185	val &= ~(1 << TRACE_TRANSITION_BIT);
0264c8c9 SRV	186	}
0264c8c9 SRV	187
0264c8c9 SRV	188	val \|= 1 << bit;
	189	current->trace_recursion = val;
	190	barrier();
	191
	192	return bit + 1;
	193	}
	194
	195	static __always_inline void trace_clear_recursion(int bit)
	196	{
0264c8c9 SRV	197	if (!bit)
	198	return;
	199
0264c8c9	200	barrier();
7b68621f SRV	201	bit--;
7b68621f SRV	202	trace_recursion_clear(bit);
0264c8c9 SRV	203	}
0264c8c9 SRV	204
6e4eb9cb SRV	205	/**
	206	* ftrace_test_recursion_trylock - tests for recursion in same context
	207	*
	208	* Use this for ftrace callbacks. This will detect if the function
	209	* tracing recursed in the same context (normal vs interrupt),
	210	*
	211	* Returns: -1 if a recursion happened.
	212	* >= 0 if no recursion
	213	*/
773c1670 SRV	214	static __always_inline int ftrace_test_recursion_trylock(unsigned long ip,
773c1670 SRV	215	unsigned long parent_ip)
6e4eb9cb	216	{
773c1670	217	return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START, TRACE_FTRACE_MAX);
6e4eb9cb SRV	218	}
	219
	220	/**
	221	* ftrace_test_recursion_unlock - called when function callback is complete
	222	* @bit: The return of a successful ftrace_test_recursion_trylock()
	223	*
	224	* This is used at the end of a ftrace callback.
	225	*/
	226	static __always_inline void ftrace_test_recursion_unlock(int bit)
	227	{
	228	trace_clear_recursion(bit);
	229	}
	230
0264c8c9 SRV	231	#endif /* CONFIG_TRACING */
0264c8c9 SRV	232	#endif /* _LINUX_TRACE_RECURSION_H */