[linux-block.git] / include / linux / compiler.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_COMPILER_H
#define __LINUX_COMPILER_H

#include <linux/compiler_types.h>

#ifndef __ASSEMBLY__

#ifdef __KERNEL__

/*
 * Note: DISABLE_BRANCH_PROFILING can be used by special lowlevel code
 * to disable branch tracing on a per file basis.
 */
#if defined(CONFIG_TRACE_BRANCH_PROFILING) \
    && !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__)
void ftrace_likely_update(struct ftrace_likely_data *f, int val,
			  int expect, int is_constant);

#define likely_notrace(x)	__builtin_expect(!!(x), 1)
#define unlikely_notrace(x)	__builtin_expect(!!(x), 0)

#define __branch_check__(x, expect, is_constant) ({			\
			long ______r;					\
			static struct ftrace_likely_data		\
				__aligned(4)				\
				__section("_ftrace_annotated_branch")	\
				______f = {				\
				.data.func = __func__,			\
				.data.file = __FILE__,			\
				.data.line = __LINE__,			\
			};						\
			______r = __builtin_expect(!!(x), expect);	\
			ftrace_likely_update(&______f, ______r,		\
					     expect, is_constant);	\
			______r;					\
		})

/*
 * Using __builtin_constant_p(x) to ignore cases where the return
 * value is always the same.  This idea is taken from a similar patch
 * written by Daniel Walker.
 */
# ifndef likely
#  define likely(x)	(__branch_check__(x, 1, __builtin_constant_p(x)))
# endif
# ifndef unlikely
#  define unlikely(x)	(__branch_check__(x, 0, __builtin_constant_p(x)))
# endif

#ifdef CONFIG_PROFILE_ALL_BRANCHES
/*
 * "Define 'is'", Bill Clinton
 * "Define 'if'", Steven Rostedt
 */
#define if(cond, ...) if ( __trace_if_var( !!(cond , ## __VA_ARGS__) ) )

#define __trace_if_var(cond) (__builtin_constant_p(cond) ? (cond) : __trace_if_value(cond))

#define __trace_if_value(cond) ({			\
	static struct ftrace_branch_data		\
		__aligned(4)				\
		__section("_ftrace_branch")		\
		__if_trace = {				\
			.func = __func__,		\
			.file = __FILE__,		\
			.line = __LINE__,		\
		};					\
	(cond) ?					\
		(__if_trace.miss_hit[1]++,1) :		\
		(__if_trace.miss_hit[0]++,0);		\
})

#endif /* CONFIG_PROFILE_ALL_BRANCHES */

#else
# define likely(x)	__builtin_expect(!!(x), 1)
# define unlikely(x)	__builtin_expect(!!(x), 0)
#endif

/* Optimization barrier */
#ifndef barrier
# define barrier() __memory_barrier()
#endif

#ifndef barrier_data
# define barrier_data(ptr) barrier()
#endif

/* workaround for GCC PR82365 if needed */
#ifndef barrier_before_unreachable
# define barrier_before_unreachable() do { } while (0)
#endif

/* Unreachable code */
#ifdef CONFIG_STACK_VALIDATION
/*
 * These macros help objtool understand GCC code flow for unreachable code.
 * The __COUNTER__ based labels are a hack to make each instance of the macros
 * unique, to convince GCC not to merge duplicate inline asm statements.
 */
#define annotate_reachable() ({						\
	asm volatile("%c0:\n\t"						\
		     ".pushsection .discard.reachable\n\t"		\
		     ".long %c0b - .\n\t"				\
		     ".popsection\n\t" : : "i" (__COUNTER__));		\
})
#define annotate_unreachable() ({					\
	asm volatile("%c0:\n\t"						\
		     ".pushsection .discard.unreachable\n\t"		\
		     ".long %c0b - .\n\t"				\
		     ".popsection\n\t" : : "i" (__COUNTER__));		\
})
#define ASM_UNREACHABLE							\
	"999:\n\t"							\
	".pushsection .discard.unreachable\n\t"				\
	".long 999b - .\n\t"						\
	".popsection\n\t"

/* Annotate a C jump table to allow objtool to follow the code flow */
#define __annotate_jump_table __section(".rodata..c_jump_table")

#else
#define annotate_reachable()
#define annotate_unreachable()
#define __annotate_jump_table
#endif

#ifndef ASM_UNREACHABLE
# define ASM_UNREACHABLE
#endif
#ifndef unreachable
# define unreachable() do {		\
	annotate_unreachable();		\
	__builtin_unreachable();	\
} while (0)
#endif

/*
 * KENTRY - kernel entry point
 * This can be used to annotate symbols (functions or data) that are used
 * without their linker symbol being referenced explicitly. For example,
 * interrupt vector handlers, or functions in the kernel image that are found
 * programatically.
 *
 * Not required for symbols exported with EXPORT_SYMBOL, or initcalls. Those
 * are handled in their own way (with KEEP() in linker scripts).
 *
 * KENTRY can be avoided if the symbols in question are marked as KEEP() in the
 * linker script. For example an architecture could KEEP() its entire
 * boot/exception vector code rather than annotate each function and data.
 */
#ifndef KENTRY
# define KENTRY(sym)						\
	extern typeof(sym) sym;					\
	static const unsigned long __kentry_##sym		\
	__used							\
	__attribute__((__section__("___kentry+" #sym)))		\
	= (unsigned long)&sym;
#endif

#ifndef RELOC_HIDE
# define RELOC_HIDE(ptr, off)					\
  ({ unsigned long __ptr;					\
     __ptr = (unsigned long) (ptr);				\
    (typeof(ptr)) (__ptr + (off)); })
#endif

#ifndef OPTIMIZER_HIDE_VAR
/* Make the optimizer believe the variable can be manipulated arbitrarily. */
#define OPTIMIZER_HIDE_VAR(var)						\
	__asm__ ("" : "=r" (var) : "0" (var))
#endif

/* Not-quite-unique ID. */
#ifndef __UNIQUE_ID
# define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __LINE__)
#endif

/**
 * data_race - mark an expression as containing intentional data races
 *
 * This data_race() macro is useful for situations in which data races
 * should be forgiven.  One example is diagnostic code that accesses
 * shared variables but is not a part of the core synchronization design.
 *
 * This macro *does not* affect normal code generation, but is a hint
 * to tooling that data races here are to be ignored.
 */
#define data_race(expr)							\
({									\
	__unqual_scalar_typeof(({ expr; })) __v = ({			\
		__kcsan_disable_current();				\
		expr;							\
	});								\
	__kcsan_enable_current();					\
	__v;								\
})

#endif /* __KERNEL__ */

/*
 * Force the compiler to emit 'sym' as a symbol, so that we can reference
 * it from inline assembler. Necessary in case 'sym' could be inlined
 * otherwise, or eliminated entirely due to lack of references that are
 * visible to the compiler.
 */
#define __ADDRESSABLE(sym) \
	static void * __section(".discard.addressable") __used \
		__UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;

/**
 * offset_to_ptr - convert a relative memory offset to an absolute pointer
 * @off:	the address of the 32-bit offset value
 */
static inline void *offset_to_ptr(const int *off)
{
	return (void *)((unsigned long)off + *off);
}

#endif /* __ASSEMBLY__ */

/* &a[0] degrades to a pointer: a different type from an array */
#define __must_be_array(a)	BUILD_BUG_ON_ZERO(__same_type((a), &(a)[0]))

/*
 * This is needed in functions which generate the stack canary, see
 * arch/x86/kernel/smpboot.c::start_secondary() for an example.
 */
#define prevent_tail_call_optimization()	mb()

#include <asm/rwonce.h>

#endif /* __LINUX_COMPILER_H */
Commit	Line	Data
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef __LINUX_COMPILER_H
	3	#define __LINUX_COMPILER_H
	4
	5	#include <linux/compiler_types.h>
	6
	7	#ifndef __ASSEMBLY__
	8
	9	#ifdef __KERNEL__
	10
	11	/*
	12	* Note: DISABLE_BRANCH_PROFILING can be used by special lowlevel code
	13	* to disable branch tracing on a per file basis.
	14	*/
	15	#if defined(CONFIG_TRACE_BRANCH_PROFILING) \
	16	&& !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__)
	17	void ftrace_likely_update(struct ftrace_likely_data *f, int val,
	18	int expect, int is_constant);
	19
	20	#define likely_notrace(x) __builtin_expect(!!(x), 1)
	21	#define unlikely_notrace(x) __builtin_expect(!!(x), 0)
	22
	23	#define __branch_check__(x, expect, is_constant) ({ \
	24	long ______r; \
	25	static struct ftrace_likely_data \
	26	__aligned(4) \
	27	__section("_ftrace_annotated_branch") \
	28	______f = { \
	29	.data.func = __func__, \
	30	.data.file = __FILE__, \
	31	.data.line = __LINE__, \
	32	}; \
	33	______r = __builtin_expect(!!(x), expect); \
	34	ftrace_likely_update(&______f, ______r, \
	35	expect, is_constant); \
	36	______r; \
	37	})
	38
	39	/*
	40	* Using __builtin_constant_p(x) to ignore cases where the return
	41	* value is always the same. This idea is taken from a similar patch
	42	* written by Daniel Walker.
	43	*/
	44	# ifndef likely
	45	# define likely(x) (__branch_check__(x, 1, __builtin_constant_p(x)))
	46	# endif
	47	# ifndef unlikely
	48	# define unlikely(x) (__branch_check__(x, 0, __builtin_constant_p(x)))
	49	# endif
	50
	51	#ifdef CONFIG_PROFILE_ALL_BRANCHES
	52	/*
	53	* "Define 'is'", Bill Clinton
	54	* "Define 'if'", Steven Rostedt
	55	*/
	56	#define if(cond, ...) if ( __trace_if_var( !!(cond , ## __VA_ARGS__) ) )
	57
	58	#define __trace_if_var(cond) (__builtin_constant_p(cond) ? (cond) : __trace_if_value(cond))
	59
	60	#define __trace_if_value(cond) ({ \
	61	static struct ftrace_branch_data \
	62	__aligned(4) \
	63	__section("_ftrace_branch") \
	64	__if_trace = { \
	65	.func = __func__, \
	66	.file = __FILE__, \
	67	.line = __LINE__, \
	68	}; \
	69	(cond) ? \
	70	(__if_trace.miss_hit[1]++,1) : \
	71	(__if_trace.miss_hit[0]++,0); \
	72	})
	73
	74	#endif /* CONFIG_PROFILE_ALL_BRANCHES */
	75
	76	#else
	77	# define likely(x) __builtin_expect(!!(x), 1)
	78	# define unlikely(x) __builtin_expect(!!(x), 0)
	79	#endif
	80
	81	/* Optimization barrier */
	82	#ifndef barrier
	83	# define barrier() __memory_barrier()
	84	#endif
	85
	86	#ifndef barrier_data
	87	# define barrier_data(ptr) barrier()
	88	#endif
	89
	90	/* workaround for GCC PR82365 if needed */
	91	#ifndef barrier_before_unreachable
	92	# define barrier_before_unreachable() do { } while (0)
	93	#endif
	94
	95	/* Unreachable code */
	96	#ifdef CONFIG_STACK_VALIDATION
	97	/*
	98	* These macros help objtool understand GCC code flow for unreachable code.
	99	* The __COUNTER__ based labels are a hack to make each instance of the macros
	100	* unique, to convince GCC not to merge duplicate inline asm statements.
	101	*/
	102	#define annotate_reachable() ({ \
	103	asm volatile("%c0:\n\t" \
	104	".pushsection .discard.reachable\n\t" \
	105	".long %c0b - .\n\t" \
	106	".popsection\n\t" : : "i" (__COUNTER__)); \
	107	})
	108	#define annotate_unreachable() ({ \
	109	asm volatile("%c0:\n\t" \
	110	".pushsection .discard.unreachable\n\t" \
	111	".long %c0b - .\n\t" \
	112	".popsection\n\t" : : "i" (__COUNTER__)); \
	113	})
	114	#define ASM_UNREACHABLE \
	115	"999:\n\t" \
	116	".pushsection .discard.unreachable\n\t" \
	117	".long 999b - .\n\t" \
	118	".popsection\n\t"
	119
	120	/* Annotate a C jump table to allow objtool to follow the code flow */
	121	#define __annotate_jump_table __section(".rodata..c_jump_table")
	122
	123	#else
	124	#define annotate_reachable()
	125	#define annotate_unreachable()
	126	#define __annotate_jump_table
	127	#endif
	128
	129	#ifndef ASM_UNREACHABLE
	130	# define ASM_UNREACHABLE
	131	#endif
	132	#ifndef unreachable
	133	# define unreachable() do { \
	134	annotate_unreachable(); \
	135	__builtin_unreachable(); \
	136	} while (0)
	137	#endif
	138
	139	/*
	140	* KENTRY - kernel entry point
	141	* This can be used to annotate symbols (functions or data) that are used
	142	* without their linker symbol being referenced explicitly. For example,
	143	* interrupt vector handlers, or functions in the kernel image that are found
	144	* programatically.
	145	*
	146	* Not required for symbols exported with EXPORT_SYMBOL, or initcalls. Those
	147	* are handled in their own way (with KEEP() in linker scripts).
	148	*
	149	* KENTRY can be avoided if the symbols in question are marked as KEEP() in the
	150	* linker script. For example an architecture could KEEP() its entire
	151	* boot/exception vector code rather than annotate each function and data.
	152	*/
	153	#ifndef KENTRY
	154	# define KENTRY(sym) \
	155	extern typeof(sym) sym; \
	156	static const unsigned long __kentry_##sym \
	157	__used \
	158	__attribute__((__section__("___kentry+" #sym))) \
	159	= (unsigned long)&sym;
	160	#endif
	161
	162	#ifndef RELOC_HIDE
	163	# define RELOC_HIDE(ptr, off) \
	164	({ unsigned long __ptr; \
	165	__ptr = (unsigned long) (ptr); \
	166	(typeof(ptr)) (__ptr + (off)); })
	167	#endif
	168
	169	#ifndef OPTIMIZER_HIDE_VAR
	170	/* Make the optimizer believe the variable can be manipulated arbitrarily. */
	171	#define OPTIMIZER_HIDE_VAR(var) \
	172	__asm__ ("" : "=r" (var) : "0" (var))
	173	#endif
	174
	175	/* Not-quite-unique ID. */
	176	#ifndef __UNIQUE_ID
	177	# define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __LINE__)
	178	#endif
	179
	180	/**
	181	* data_race - mark an expression as containing intentional data races
	182	*
	183	* This data_race() macro is useful for situations in which data races
	184	* should be forgiven. One example is diagnostic code that accesses
	185	* shared variables but is not a part of the core synchronization design.
	186	*
	187	* This macro does not affect normal code generation, but is a hint
	188	* to tooling that data races here are to be ignored.
	189	*/
	190	#define data_race(expr) \
	191	({ \
	192	__unqual_scalar_typeof(({ expr; })) __v = ({ \
	193	__kcsan_disable_current(); \
	194	expr; \
	195	}); \
	196	__kcsan_enable_current(); \
	197	__v; \
	198	})
	199
	200	#endif /* __KERNEL__ */
	201
	202	/*
	203	* Force the compiler to emit 'sym' as a symbol, so that we can reference
	204	* it from inline assembler. Necessary in case 'sym' could be inlined
	205	* otherwise, or eliminated entirely due to lack of references that are
	206	* visible to the compiler.
	207	*/
	208	#define __ADDRESSABLE(sym) \
	209	static void * __section(".discard.addressable") __used \
	210	__UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;
	211
	212	/**
	213	* offset_to_ptr - convert a relative memory offset to an absolute pointer
	214	* @off: the address of the 32-bit offset value
	215	*/
	216	static inline void offset_to_ptr(const int off)
	217	{
	218	return (void )((unsigned long)off + off);
	219	}
	220
	221	#endif /* __ASSEMBLY__ */
	222
	223	/* &a[0] degrades to a pointer: a different type from an array */
	224	#define __must_be_array(a) BUILD_BUG_ON_ZERO(__same_type((a), &(a)[0]))
	225
	226	/*
	227	* This is needed in functions which generate the stack canary, see
	228	* arch/x86/kernel/smpboot.c::start_secondary() for an example.
	229	*/
	230	#define prevent_tail_call_optimization() mb()
	231
	232	#include <asm/rwonce.h>
	233
	234	#endif /* __LINUX_COMPILER_H */