[linux-block.git] / include / linux / kcsan-checks.h

/* SPDX-License-Identifier: GPL-2.0 */

#ifndef _LINUX_KCSAN_CHECKS_H
#define _LINUX_KCSAN_CHECKS_H

/* Note: Only include what is already included by compiler.h. */
#include <linux/compiler_attributes.h>
#include <linux/types.h>

/*
 * ACCESS TYPE MODIFIERS
 *
 *   <none>: normal read access;
 *   WRITE : write access;
 *   ATOMIC: access is atomic;
 *   ASSERT: access is not a regular access, but an assertion;
 *   SCOPED: access is a scoped access;
 */
#define KCSAN_ACCESS_WRITE  0x1
#define KCSAN_ACCESS_ATOMIC 0x2
#define KCSAN_ACCESS_ASSERT 0x4
#define KCSAN_ACCESS_SCOPED 0x8

/*
 * __kcsan_*: Always calls into the runtime when KCSAN is enabled. This may be used
 * even in compilation units that selectively disable KCSAN, but must use KCSAN
 * to validate access to an address. Never use these in header files!
 */
#ifdef CONFIG_KCSAN
/**
 * __kcsan_check_access - check generic access for races
 *
 * @ptr: address of access
 * @size: size of access
 * @type: access type modifier
 */
void __kcsan_check_access(const volatile void *ptr, size_t size, int type);

/**
 * kcsan_disable_current - disable KCSAN for the current context
 *
 * Supports nesting.
 */
void kcsan_disable_current(void);

/**
 * kcsan_enable_current - re-enable KCSAN for the current context
 *
 * Supports nesting.
 */
void kcsan_enable_current(void);
void kcsan_enable_current_nowarn(void); /* Safe in uaccess regions. */

/**
 * kcsan_nestable_atomic_begin - begin nestable atomic region
 *
 * Accesses within the atomic region may appear to race with other accesses but
 * should be considered atomic.
 */
void kcsan_nestable_atomic_begin(void);

/**
 * kcsan_nestable_atomic_end - end nestable atomic region
 */
void kcsan_nestable_atomic_end(void);

/**
 * kcsan_flat_atomic_begin - begin flat atomic region
 *
 * Accesses within the atomic region may appear to race with other accesses but
 * should be considered atomic.
 */
void kcsan_flat_atomic_begin(void);

/**
 * kcsan_flat_atomic_end - end flat atomic region
 */
void kcsan_flat_atomic_end(void);

/**
 * kcsan_atomic_next - consider following accesses as atomic
 *
 * Force treating the next n memory accesses for the current context as atomic
 * operations.
 *
 * @n: number of following memory accesses to treat as atomic.
 */
void kcsan_atomic_next(int n);

/**
 * kcsan_set_access_mask - set access mask
 *
 * Set the access mask for all accesses for the current context if non-zero.
 * Only value changes to bits set in the mask will be reported.
 *
 * @mask: bitmask
 */
void kcsan_set_access_mask(unsigned long mask);

/* Scoped access information. */
struct kcsan_scoped_access {
	struct list_head list;
	const volatile void *ptr;
	size_t size;
	int type;
};
/*
 * Automatically call kcsan_end_scoped_access() when kcsan_scoped_access goes
 * out of scope; relies on attribute "cleanup", which is supported by all
 * compilers that support KCSAN.
 */
#define __kcsan_cleanup_scoped                                                 \
	__maybe_unused __attribute__((__cleanup__(kcsan_end_scoped_access)))

/**
 * kcsan_begin_scoped_access - begin scoped access
 *
 * Begin scoped access and initialize @sa, which will cause KCSAN to
 * continuously check the memory range in the current thread until
 * kcsan_end_scoped_access() is called for @sa.
 *
 * Scoped accesses are implemented by appending @sa to an internal list for the
 * current execution context, and then checked on every call into the KCSAN
 * runtime.
 *
 * @ptr: address of access
 * @size: size of access
 * @type: access type modifier
 * @sa: struct kcsan_scoped_access to use for the scope of the access
 */
struct kcsan_scoped_access *
kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type,
			  struct kcsan_scoped_access *sa);

/**
 * kcsan_end_scoped_access - end scoped access
 *
 * End a scoped access, which will stop KCSAN checking the memory range.
 * Requires that kcsan_begin_scoped_access() was previously called once for @sa.
 *
 * @sa: a previously initialized struct kcsan_scoped_access
 */
void kcsan_end_scoped_access(struct kcsan_scoped_access *sa);


#else /* CONFIG_KCSAN */

static inline void __kcsan_check_access(const volatile void *ptr, size_t size,
					int type) { }

static inline void kcsan_disable_current(void)		{ }
static inline void kcsan_enable_current(void)		{ }
static inline void kcsan_enable_current_nowarn(void)	{ }
static inline void kcsan_nestable_atomic_begin(void)	{ }
static inline void kcsan_nestable_atomic_end(void)	{ }
static inline void kcsan_flat_atomic_begin(void)	{ }
static inline void kcsan_flat_atomic_end(void)		{ }
static inline void kcsan_atomic_next(int n)		{ }
static inline void kcsan_set_access_mask(unsigned long mask) { }

struct kcsan_scoped_access { };
#define __kcsan_cleanup_scoped __maybe_unused
static inline struct kcsan_scoped_access *
kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type,
			  struct kcsan_scoped_access *sa) { return sa; }
static inline void kcsan_end_scoped_access(struct kcsan_scoped_access *sa) { }

#endif /* CONFIG_KCSAN */

#ifdef __SANITIZE_THREAD__
/*
 * Only calls into the runtime when the particular compilation unit has KCSAN
 * instrumentation enabled. May be used in header files.
 */
#define kcsan_check_access __kcsan_check_access

/*
 * Only use these to disable KCSAN for accesses in the current compilation unit;
 * calls into libraries may still perform KCSAN checks.
 */
#define __kcsan_disable_current kcsan_disable_current
#define __kcsan_enable_current kcsan_enable_current_nowarn
#else
static inline void kcsan_check_access(const volatile void *ptr, size_t size,
				      int type) { }
static inline void __kcsan_enable_current(void)  { }
static inline void __kcsan_disable_current(void) { }
#endif

/**
 * __kcsan_check_read - check regular read access for races
 *
 * @ptr: address of access
 * @size: size of access
 */
#define __kcsan_check_read(ptr, size) __kcsan_check_access(ptr, size, 0)

/**
 * __kcsan_check_write - check regular write access for races
 *
 * @ptr: address of access
 * @size: size of access
 */
#define __kcsan_check_write(ptr, size)                                         \
	__kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)

/**
 * kcsan_check_read - check regular read access for races
 *
 * @ptr: address of access
 * @size: size of access
 */
#define kcsan_check_read(ptr, size) kcsan_check_access(ptr, size, 0)

/**
 * kcsan_check_write - check regular write access for races
 *
 * @ptr: address of access
 * @size: size of access
 */
#define kcsan_check_write(ptr, size)                                           \
	kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)

/*
 * Check for atomic accesses: if atomic accesses are not ignored, this simply
 * aliases to kcsan_check_access(), otherwise becomes a no-op.
 */
#ifdef CONFIG_KCSAN_IGNORE_ATOMICS
#define kcsan_check_atomic_read(...)	do { } while (0)
#define kcsan_check_atomic_write(...)	do { } while (0)
#else
#define kcsan_check_atomic_read(ptr, size)                                     \
	kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC)
#define kcsan_check_atomic_write(ptr, size)                                    \
	kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE)
#endif

/**
 * ASSERT_EXCLUSIVE_WRITER - assert no concurrent writes to @var
 *
 * Assert that there are no concurrent writes to @var; other readers are
 * allowed. This assertion can be used to specify properties of concurrent code,
 * where violation cannot be detected as a normal data race.
 *
 * For example, if we only have a single writer, but multiple concurrent
 * readers, to avoid data races, all these accesses must be marked; even
 * concurrent marked writes racing with the single writer are bugs.
 * Unfortunately, due to being marked, they are no longer data races. For cases
 * like these, we can use the macro as follows:
 *
 * .. code-block:: c
 *
 *	void writer(void) {
 *		spin_lock(&update_foo_lock);
 *		ASSERT_EXCLUSIVE_WRITER(shared_foo);
 *		WRITE_ONCE(shared_foo, ...);
 *		spin_unlock(&update_foo_lock);
 *	}
 *	void reader(void) {
 *		// update_foo_lock does not need to be held!
 *		... = READ_ONCE(shared_foo);
 *	}
 *
 * Note: ASSERT_EXCLUSIVE_WRITER_SCOPED(), if applicable, performs more thorough
 * checking if a clear scope where no concurrent writes are expected exists.
 *
 * @var: variable to assert on
 */
#define ASSERT_EXCLUSIVE_WRITER(var)                                           \
	__kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)

/*
 * Helper macros for implementation of for ASSERT_EXCLUSIVE_*_SCOPED(). @id is
 * expected to be unique for the scope in which instances of kcsan_scoped_access
 * are declared.
 */
#define __kcsan_scoped_name(c, suffix) __kcsan_scoped_##c##suffix
#define __ASSERT_EXCLUSIVE_SCOPED(var, type, id)                               \
	struct kcsan_scoped_access __kcsan_scoped_name(id, _)                  \
		__kcsan_cleanup_scoped;                                        \
	struct kcsan_scoped_access *__kcsan_scoped_name(id, _dummy_p)          \
		__maybe_unused = kcsan_begin_scoped_access(                    \
			&(var), sizeof(var), KCSAN_ACCESS_SCOPED | (type),     \
			&__kcsan_scoped_name(id, _))

/**
 * ASSERT_EXCLUSIVE_WRITER_SCOPED - assert no concurrent writes to @var in scope
 *
 * Scoped variant of ASSERT_EXCLUSIVE_WRITER().
 *
 * Assert that there are no concurrent writes to @var for the duration of the
 * scope in which it is introduced. This provides a better way to fully cover
 * the enclosing scope, compared to multiple ASSERT_EXCLUSIVE_WRITER(), and
 * increases the likelihood for KCSAN to detect racing accesses.
 *
 * For example, it allows finding race-condition bugs that only occur due to
 * state changes within the scope itself:
 *
 * .. code-block:: c
 *
 *	void writer(void) {
 *		spin_lock(&update_foo_lock);
 *		{
 *			ASSERT_EXCLUSIVE_WRITER_SCOPED(shared_foo);
 *			WRITE_ONCE(shared_foo, 42);
 *			...
 *			// shared_foo should still be 42 here!
 *		}
 *		spin_unlock(&update_foo_lock);
 *	}
 *	void buggy(void) {
 *		if (READ_ONCE(shared_foo) == 42)
 *			WRITE_ONCE(shared_foo, 1); // bug!
 *	}
 *
 * @var: variable to assert on
 */
#define ASSERT_EXCLUSIVE_WRITER_SCOPED(var)                                    \
	__ASSERT_EXCLUSIVE_SCOPED(var, KCSAN_ACCESS_ASSERT, __COUNTER__)

/**
 * ASSERT_EXCLUSIVE_ACCESS - assert no concurrent accesses to @var
 *
 * Assert that there are no concurrent accesses to @var (no readers nor
 * writers). This assertion can be used to specify properties of concurrent
 * code, where violation cannot be detected as a normal data race.
 *
 * For example, where exclusive access is expected after determining no other
 * users of an object are left, but the object is not actually freed. We can
 * check that this property actually holds as follows:
 *
 * .. code-block:: c
 *
 *	if (refcount_dec_and_test(&obj->refcnt)) {
 *		ASSERT_EXCLUSIVE_ACCESS(*obj);
 *		do_some_cleanup(obj);
 *		release_for_reuse(obj);
 *	}
 *
 * Note:
 *
 * 1. ASSERT_EXCLUSIVE_ACCESS_SCOPED(), if applicable, performs more thorough
 *    checking if a clear scope where no concurrent accesses are expected exists.
 *
 * 2. For cases where the object is freed, `KASAN <kasan.html>`_ is a better
 *    fit to detect use-after-free bugs.
 *
 * @var: variable to assert on
 */
#define ASSERT_EXCLUSIVE_ACCESS(var)                                           \
	__kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT)

/**
 * ASSERT_EXCLUSIVE_ACCESS_SCOPED - assert no concurrent accesses to @var in scope
 *
 * Scoped variant of ASSERT_EXCLUSIVE_ACCESS().
 *
 * Assert that there are no concurrent accesses to @var (no readers nor writers)
 * for the entire duration of the scope in which it is introduced. This provides
 * a better way to fully cover the enclosing scope, compared to multiple
 * ASSERT_EXCLUSIVE_ACCESS(), and increases the likelihood for KCSAN to detect
 * racing accesses.
 *
 * @var: variable to assert on
 */
#define ASSERT_EXCLUSIVE_ACCESS_SCOPED(var)                                    \
	__ASSERT_EXCLUSIVE_SCOPED(var, KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT, __COUNTER__)

/**
 * ASSERT_EXCLUSIVE_BITS - assert no concurrent writes to subset of bits in @var
 *
 * Bit-granular variant of ASSERT_EXCLUSIVE_WRITER().
 *
 * Assert that there are no concurrent writes to a subset of bits in @var;
 * concurrent readers are permitted. This assertion captures more detailed
 * bit-level properties, compared to the other (word granularity) assertions.
 * Only the bits set in @mask are checked for concurrent modifications, while
 * ignoring the remaining bits, i.e. concurrent writes (or reads) to ~mask bits
 * are ignored.
 *
 * Use this for variables, where some bits must not be modified concurrently,
 * yet other bits are expected to be modified concurrently.
 *
 * For example, variables where, after initialization, some bits are read-only,
 * but other bits may still be modified concurrently. A reader may wish to
 * assert that this is true as follows:
 *
 * .. code-block:: c
 *
 *	ASSERT_EXCLUSIVE_BITS(flags, READ_ONLY_MASK);
 *	foo = (READ_ONCE(flags) & READ_ONLY_MASK) >> READ_ONLY_SHIFT;
 *
 * Note: The access that immediately follows ASSERT_EXCLUSIVE_BITS() is assumed
 * to access the masked bits only, and KCSAN optimistically assumes it is
 * therefore safe, even in the presence of data races, and marking it with
 * READ_ONCE() is optional from KCSAN's point-of-view. We caution, however, that
 * it may still be advisable to do so, since we cannot reason about all compiler
 * optimizations when it comes to bit manipulations (on the reader and writer
 * side). If you are sure nothing can go wrong, we can write the above simply
 * as:
 *
 * .. code-block:: c
 *
 *	ASSERT_EXCLUSIVE_BITS(flags, READ_ONLY_MASK);
 *	foo = (flags & READ_ONLY_MASK) >> READ_ONLY_SHIFT;
 *
 * Another example, where this may be used, is when certain bits of @var may
 * only be modified when holding the appropriate lock, but other bits may still
 * be modified concurrently. Writers, where other bits may change concurrently,
 * could use the assertion as follows:
 *
 * .. code-block:: c
 *
 *	spin_lock(&foo_lock);
 *	ASSERT_EXCLUSIVE_BITS(flags, FOO_MASK);
 *	old_flags = flags;
 *	new_flags = (old_flags & ~FOO_MASK) | (new_foo << FOO_SHIFT);
 *	if (cmpxchg(&flags, old_flags, new_flags) != old_flags) { ... }
 *	spin_unlock(&foo_lock);
 *
 * @var: variable to assert on
 * @mask: only check for modifications to bits set in @mask
 */
#define ASSERT_EXCLUSIVE_BITS(var, mask)                                       \
	do {                                                                   \
		kcsan_set_access_mask(mask);                                   \
		__kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT);\
		kcsan_set_access_mask(0);                                      \
		kcsan_atomic_next(1);                                          \
	} while (0)

#endif /* _LINUX_KCSAN_CHECKS_H */
Commit	Line	Data
dfd402a4 ME	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	#ifndef _LINUX_KCSAN_CHECKS_H
	4	#define _LINUX_KCSAN_CHECKS_H
	5
757a4cef ME	6	/* Note: Only include what is already included by compiler.h. */
757a4cef ME	7	#include <linux/compiler_attributes.h>
dfd402a4 ME	8	#include <linux/types.h>
	9
	10	/*
d591ec3d ME	11	* ACCESS TYPE MODIFIERS
	12	*
	13	* <none>: normal read access;
	14	* WRITE : write access;
	15	* ATOMIC: access is atomic;
	16	* ASSERT: access is not a regular access, but an assertion;
757a4cef	17	* SCOPED: access is a scoped access;
dfd402a4	18	*/
5cbaefe9	19	#define KCSAN_ACCESS_WRITE 0x1
dfd402a4	20	#define KCSAN_ACCESS_ATOMIC 0x2
d591ec3d	21	#define KCSAN_ACCESS_ASSERT 0x4
757a4cef	22	#define KCSAN_ACCESS_SCOPED 0x8
dfd402a4 ME	23
dfd402a4 ME	24	/*
5cbaefe9	25	* __kcsan_*: Always calls into the runtime when KCSAN is enabled. This may be used
dfd402a4	26	* even in compilation units that selectively disable KCSAN, but must use KCSAN
5cbaefe9	27	* to validate access to an address. Never use these in header files!
dfd402a4 ME	28	*/
	29	#ifdef CONFIG_KCSAN
	30	/**
d591ec3d	31	* __kcsan_check_access - check generic access for races
dfd402a4	32	*
1443b8c9 ME	33	* @ptr: address of access
	34	* @size: size of access
	35	* @type: access type modifier
dfd402a4 ME	36	*/
	37	void __kcsan_check_access(const volatile void *ptr, size_t size, int type);
	38
01b4ff58 ME	39	/**
	40	* kcsan_disable_current - disable KCSAN for the current context
	41	*
	42	* Supports nesting.
	43	*/
	44	void kcsan_disable_current(void);
	45
	46	/**
	47	* kcsan_enable_current - re-enable KCSAN for the current context
	48	*
	49	* Supports nesting.
	50	*/
	51	void kcsan_enable_current(void);
19acd03d	52	void kcsan_enable_current_nowarn(void); /* Safe in uaccess regions. */
01b4ff58	53
f0f6928c ME	54	/**
	55	* kcsan_nestable_atomic_begin - begin nestable atomic region
	56	*
	57	* Accesses within the atomic region may appear to race with other accesses but
	58	* should be considered atomic.
	59	*/
	60	void kcsan_nestable_atomic_begin(void);
	61
	62	/**
	63	* kcsan_nestable_atomic_end - end nestable atomic region
	64	*/
	65	void kcsan_nestable_atomic_end(void);
	66
	67	/**
	68	* kcsan_flat_atomic_begin - begin flat atomic region
	69	*
	70	* Accesses within the atomic region may appear to race with other accesses but
	71	* should be considered atomic.
	72	*/
	73	void kcsan_flat_atomic_begin(void);
	74
	75	/**
	76	* kcsan_flat_atomic_end - end flat atomic region
	77	*/
	78	void kcsan_flat_atomic_end(void);
	79
	80	/**
	81	* kcsan_atomic_next - consider following accesses as atomic
	82	*
	83	* Force treating the next n memory accesses for the current context as atomic
	84	* operations.
	85	*
1443b8c9	86	* @n: number of following memory accesses to treat as atomic.
f0f6928c ME	87	*/
	88	void kcsan_atomic_next(int n);
	89
81af89e1 ME	90	/**
	91	* kcsan_set_access_mask - set access mask
	92	*
	93	* Set the access mask for all accesses for the current context if non-zero.
	94	* Only value changes to bits set in the mask will be reported.
	95	*
1443b8c9	96	* @mask: bitmask
81af89e1 ME	97	*/
	98	void kcsan_set_access_mask(unsigned long mask);
	99
757a4cef ME	100	/* Scoped access information. */
	101	struct kcsan_scoped_access {
	102	struct list_head list;
	103	const volatile void *ptr;
	104	size_t size;
	105	int type;
	106	};
	107	/*
	108	* Automatically call kcsan_end_scoped_access() when kcsan_scoped_access goes
	109	* out of scope; relies on attribute "cleanup", which is supported by all
	110	* compilers that support KCSAN.
	111	*/
	112	#define __kcsan_cleanup_scoped \
	113	__maybe_unused __attribute__((__cleanup__(kcsan_end_scoped_access)))
	114
	115	/**
	116	* kcsan_begin_scoped_access - begin scoped access
	117	*
	118	* Begin scoped access and initialize @sa, which will cause KCSAN to
	119	* continuously check the memory range in the current thread until
	120	* kcsan_end_scoped_access() is called for @sa.
	121	*
	122	* Scoped accesses are implemented by appending @sa to an internal list for the
	123	* current execution context, and then checked on every call into the KCSAN
	124	* runtime.
	125	*
	126	* @ptr: address of access
	127	* @size: size of access
	128	* @type: access type modifier
	129	* @sa: struct kcsan_scoped_access to use for the scope of the access
	130	*/
	131	struct kcsan_scoped_access *
	132	kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type,
	133	struct kcsan_scoped_access *sa);
	134
	135	/**
	136	* kcsan_end_scoped_access - end scoped access
	137	*
	138	* End a scoped access, which will stop KCSAN checking the memory range.
	139	* Requires that kcsan_begin_scoped_access() was previously called once for @sa.
	140	*
	141	* @sa: a previously initialized struct kcsan_scoped_access
	142	*/
	143	void kcsan_end_scoped_access(struct kcsan_scoped_access *sa);
	144
	145
f0f6928c ME	146	#else /* CONFIG_KCSAN */
f0f6928c ME	147
dfd402a4 ME	148	static inline void __kcsan_check_access(const volatile void *ptr, size_t size,
dfd402a4 ME	149	int type) { }
f0f6928c	150
01b4ff58 ME	151	static inline void kcsan_disable_current(void) { }
01b4ff58 ME	152	static inline void kcsan_enable_current(void) { }
19acd03d	153	static inline void kcsan_enable_current_nowarn(void) { }
f0f6928c ME	154	static inline void kcsan_nestable_atomic_begin(void) { }
	155	static inline void kcsan_nestable_atomic_end(void) { }
	156	static inline void kcsan_flat_atomic_begin(void) { }
	157	static inline void kcsan_flat_atomic_end(void) { }
	158	static inline void kcsan_atomic_next(int n) { }
81af89e1	159	static inline void kcsan_set_access_mask(unsigned long mask) { }
f0f6928c	160
757a4cef ME	161	struct kcsan_scoped_access { };
	162	#define __kcsan_cleanup_scoped __maybe_unused
	163	static inline struct kcsan_scoped_access *
	164	kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type,
	165	struct kcsan_scoped_access *sa) { return sa; }
	166	static inline void kcsan_end_scoped_access(struct kcsan_scoped_access *sa) { }
	167
f0f6928c	168	#endif /* CONFIG_KCSAN */
dfd402a4	169
19acd03d	170	#ifdef __SANITIZE_THREAD__
dfd402a4	171	/*
19acd03d ME	172	* Only calls into the runtime when the particular compilation unit has KCSAN
19acd03d ME	173	* instrumentation enabled. May be used in header files.
dfd402a4	174	*/
dfd402a4	175	#define kcsan_check_access __kcsan_check_access
19acd03d ME	176
	177	/*
	178	* Only use these to disable KCSAN for accesses in the current compilation unit;
	179	* calls into libraries may still perform KCSAN checks.
	180	*/
	181	#define __kcsan_disable_current kcsan_disable_current
	182	#define __kcsan_enable_current kcsan_enable_current_nowarn
dfd402a4 ME	183	#else
	184	static inline void kcsan_check_access(const volatile void *ptr, size_t size,
	185	int type) { }
19acd03d ME	186	static inline void __kcsan_enable_current(void) { }
19acd03d ME	187	static inline void __kcsan_disable_current(void) { }
dfd402a4 ME	188	#endif
	189
	190	/**
d591ec3d	191	* __kcsan_check_read - check regular read access for races
dfd402a4	192	*
1443b8c9 ME	193	* @ptr: address of access
1443b8c9 ME	194	* @size: size of access
dfd402a4 ME	195	*/
	196	#define __kcsan_check_read(ptr, size) __kcsan_check_access(ptr, size, 0)
	197
	198	/**
d591ec3d	199	* __kcsan_check_write - check regular write access for races
dfd402a4	200	*
1443b8c9 ME	201	* @ptr: address of access
1443b8c9 ME	202	* @size: size of access
dfd402a4 ME	203	*/
	204	#define __kcsan_check_write(ptr, size) \
	205	__kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
	206
	207	/**
d591ec3d	208	* kcsan_check_read - check regular read access for races
dfd402a4	209	*
1443b8c9 ME	210	* @ptr: address of access
1443b8c9 ME	211	* @size: size of access
dfd402a4 ME	212	*/
	213	#define kcsan_check_read(ptr, size) kcsan_check_access(ptr, size, 0)
	214
	215	/**
d591ec3d	216	* kcsan_check_write - check regular write access for races
dfd402a4	217	*
1443b8c9 ME	218	* @ptr: address of access
1443b8c9 ME	219	* @size: size of access
dfd402a4 ME	220	*/
	221	#define kcsan_check_write(ptr, size) \
	222	kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
	223
	224	/*
5cbaefe9 IM	225	* Check for atomic accesses: if atomic accesses are not ignored, this simply
5cbaefe9 IM	226	* aliases to kcsan_check_access(), otherwise becomes a no-op.
dfd402a4 ME	227	*/
dfd402a4 ME	228	#ifdef CONFIG_KCSAN_IGNORE_ATOMICS
5cbaefe9 IM	229	#define kcsan_check_atomic_read(...) do { } while (0)
5cbaefe9 IM	230	#define kcsan_check_atomic_write(...) do { } while (0)
dfd402a4 ME	231	#else
	232	#define kcsan_check_atomic_read(ptr, size) \
	233	kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC)
	234	#define kcsan_check_atomic_write(ptr, size) \
	235	kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC \| KCSAN_ACCESS_WRITE)
	236	#endif
	237
f97f713d	238	/**
703b3215	239	* ASSERT_EXCLUSIVE_WRITER - assert no concurrent writes to @var
f97f713d	240	*
703b3215	241	* Assert that there are no concurrent writes to @var; other readers are
f97f713d ME	242	* allowed. This assertion can be used to specify properties of concurrent code,
	243	* where violation cannot be detected as a normal data race.
	244	*
1443b8c9 ME	245	* For example, if we only have a single writer, but multiple concurrent
	246	* readers, to avoid data races, all these accesses must be marked; even
	247	* concurrent marked writes racing with the single writer are bugs.
	248	* Unfortunately, due to being marked, they are no longer data races. For cases
	249	* like these, we can use the macro as follows:
f97f713d	250	*
1443b8c9 ME	251	* .. code-block:: c
	252	*
	253	* void writer(void) {
	254	* spin_lock(&update_foo_lock);
	255	* ASSERT_EXCLUSIVE_WRITER(shared_foo);
	256	* WRITE_ONCE(shared_foo, ...);
	257	* spin_unlock(&update_foo_lock);
	258	* }
	259	* void reader(void) {
	260	* // update_foo_lock does not need to be held!
	261	* ... = READ_ONCE(shared_foo);
	262	* }
	263	*
d8949ef1 ME	264	* Note: ASSERT_EXCLUSIVE_WRITER_SCOPED(), if applicable, performs more thorough
	265	* checking if a clear scope where no concurrent writes are expected exists.
	266	*
1443b8c9	267	* @var: variable to assert on
f97f713d ME	268	*/
	269	#define ASSERT_EXCLUSIVE_WRITER(var) \
	270	__kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
	271
d8949ef1 ME	272	/*
	273	* Helper macros for implementation of for ASSERT_EXCLUSIVE_*_SCOPED(). @id is
	274	* expected to be unique for the scope in which instances of kcsan_scoped_access
	275	* are declared.
	276	*/
	277	#define __kcsan_scoped_name(c, suffix) __kcsan_scoped_##c##suffix
	278	#define __ASSERT_EXCLUSIVE_SCOPED(var, type, id) \
	279	struct kcsan_scoped_access __kcsan_scoped_name(id, _) \
	280	__kcsan_cleanup_scoped; \
	281	struct kcsan_scoped_access *__kcsan_scoped_name(id, _dummy_p) \
	282	__maybe_unused = kcsan_begin_scoped_access( \
	283	&(var), sizeof(var), KCSAN_ACCESS_SCOPED \| (type), \
	284	&__kcsan_scoped_name(id, _))
	285
	286	/**
	287	* ASSERT_EXCLUSIVE_WRITER_SCOPED - assert no concurrent writes to @var in scope
	288	*
	289	* Scoped variant of ASSERT_EXCLUSIVE_WRITER().
	290	*
	291	* Assert that there are no concurrent writes to @var for the duration of the
	292	* scope in which it is introduced. This provides a better way to fully cover
	293	* the enclosing scope, compared to multiple ASSERT_EXCLUSIVE_WRITER(), and
	294	* increases the likelihood for KCSAN to detect racing accesses.
	295	*
	296	* For example, it allows finding race-condition bugs that only occur due to
	297	* state changes within the scope itself:
	298	*
	299	* .. code-block:: c
	300	*
	301	* void writer(void) {
	302	* spin_lock(&update_foo_lock);
	303	* {
	304	* ASSERT_EXCLUSIVE_WRITER_SCOPED(shared_foo);
	305	* WRITE_ONCE(shared_foo, 42);
	306	* ...
	307	* // shared_foo should still be 42 here!
	308	* }
	309	* spin_unlock(&update_foo_lock);
	310	* }
	311	* void buggy(void) {
	312	* if (READ_ONCE(shared_foo) == 42)
	313	* WRITE_ONCE(shared_foo, 1); // bug!
	314	* }
	315	*
	316	* @var: variable to assert on
	317	*/
	318	#define ASSERT_EXCLUSIVE_WRITER_SCOPED(var) \
	319	__ASSERT_EXCLUSIVE_SCOPED(var, KCSAN_ACCESS_ASSERT, __COUNTER__)
	320
f97f713d	321	/**
703b3215	322	* ASSERT_EXCLUSIVE_ACCESS - assert no concurrent accesses to @var
f97f713d	323	*
703b3215 ME	324	* Assert that there are no concurrent accesses to @var (no readers nor
	325	* writers). This assertion can be used to specify properties of concurrent
	326	* code, where violation cannot be detected as a normal data race.
f97f713d	327	*
1443b8c9 ME	328	* For example, where exclusive access is expected after determining no other
	329	* users of an object are left, but the object is not actually freed. We can
	330	* check that this property actually holds as follows:
	331	*
	332	* .. code-block:: c
f97f713d ME	333	*
	334	* if (refcount_dec_and_test(&obj->refcnt)) {
	335	* ASSERT_EXCLUSIVE_ACCESS(*obj);
1443b8c9 ME	336	* do_some_cleanup(obj);
1443b8c9 ME	337	* release_for_reuse(obj);
f97f713d ME	338	* }
f97f713d ME	339	*
15d737f8	340	* Note:
d8949ef1	341	*
15d737f8 MCC	342	* 1. ASSERT_EXCLUSIVE_ACCESS_SCOPED(), if applicable, performs more thorough
	343	* checking if a clear scope where no concurrent accesses are expected exists.
	344	*
	345	* 2. For cases where the object is freed, `KASAN <kasan.html>`_ is a better
	346	* fit to detect use-after-free bugs.
1443b8c9 ME	347	*
1443b8c9 ME	348	* @var: variable to assert on
f97f713d ME	349	*/
	350	#define ASSERT_EXCLUSIVE_ACCESS(var) \
	351	__kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_WRITE \| KCSAN_ACCESS_ASSERT)
	352
d8949ef1 ME	353	/**
	354	* ASSERT_EXCLUSIVE_ACCESS_SCOPED - assert no concurrent accesses to @var in scope
	355	*
	356	* Scoped variant of ASSERT_EXCLUSIVE_ACCESS().
	357	*
	358	* Assert that there are no concurrent accesses to @var (no readers nor writers)
	359	* for the entire duration of the scope in which it is introduced. This provides
	360	* a better way to fully cover the enclosing scope, compared to multiple
	361	* ASSERT_EXCLUSIVE_ACCESS(), and increases the likelihood for KCSAN to detect
	362	* racing accesses.
	363	*
	364	* @var: variable to assert on
	365	*/
	366	#define ASSERT_EXCLUSIVE_ACCESS_SCOPED(var) \
	367	__ASSERT_EXCLUSIVE_SCOPED(var, KCSAN_ACCESS_WRITE \| KCSAN_ACCESS_ASSERT, __COUNTER__)
	368
703b3215 ME	369	/**
	370	* ASSERT_EXCLUSIVE_BITS - assert no concurrent writes to subset of bits in @var
	371	*
d8949ef1	372	* Bit-granular variant of ASSERT_EXCLUSIVE_WRITER().
703b3215 ME	373	*
	374	* Assert that there are no concurrent writes to a subset of bits in @var;
	375	* concurrent readers are permitted. This assertion captures more detailed
	376	* bit-level properties, compared to the other (word granularity) assertions.
	377	* Only the bits set in @mask are checked for concurrent modifications, while
1443b8c9	378	* ignoring the remaining bits, i.e. concurrent writes (or reads) to ~mask bits
703b3215 ME	379	* are ignored.
	380	*
	381	* Use this for variables, where some bits must not be modified concurrently,
	382	* yet other bits are expected to be modified concurrently.
	383	*
	384	* For example, variables where, after initialization, some bits are read-only,
	385	* but other bits may still be modified concurrently. A reader may wish to
	386	* assert that this is true as follows:
	387	*
1443b8c9 ME	388	* .. code-block:: c
1443b8c9 ME	389	*
703b3215 ME	390	* ASSERT_EXCLUSIVE_BITS(flags, READ_ONLY_MASK);
	391	* foo = (READ_ONCE(flags) & READ_ONLY_MASK) >> READ_ONLY_SHIFT;
	392	*
1443b8c9 ME	393	* Note: The access that immediately follows ASSERT_EXCLUSIVE_BITS() is assumed
	394	* to access the masked bits only, and KCSAN optimistically assumes it is
	395	* therefore safe, even in the presence of data races, and marking it with
	396	* READ_ONCE() is optional from KCSAN's point-of-view. We caution, however, that
	397	* it may still be advisable to do so, since we cannot reason about all compiler
	398	* optimizations when it comes to bit manipulations (on the reader and writer
	399	* side). If you are sure nothing can go wrong, we can write the above simply
	400	* as:
	401	*
	402	* .. code-block:: c
703b3215 ME	403	*
	404	* ASSERT_EXCLUSIVE_BITS(flags, READ_ONLY_MASK);
	405	* foo = (flags & READ_ONLY_MASK) >> READ_ONLY_SHIFT;
	406	*
	407	* Another example, where this may be used, is when certain bits of @var may
	408	* only be modified when holding the appropriate lock, but other bits may still
	409	* be modified concurrently. Writers, where other bits may change concurrently,
	410	* could use the assertion as follows:
	411	*
1443b8c9 ME	412	* .. code-block:: c
1443b8c9 ME	413	*
703b3215 ME	414	* spin_lock(&foo_lock);
703b3215 ME	415	* ASSERT_EXCLUSIVE_BITS(flags, FOO_MASK);
1443b8c9	416	* old_flags = flags;
703b3215 ME	417	* new_flags = (old_flags & ~FOO_MASK) \| (new_foo << FOO_SHIFT);
	418	* if (cmpxchg(&flags, old_flags, new_flags) != old_flags) { ... }
	419	* spin_unlock(&foo_lock);
	420	*
1443b8c9 ME	421	* @var: variable to assert on
1443b8c9 ME	422	* @mask: only check for modifications to bits set in @mask
703b3215 ME	423	*/
	424	#define ASSERT_EXCLUSIVE_BITS(var, mask) \
	425	do { \
	426	kcsan_set_access_mask(mask); \
	427	__kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT);\
	428	kcsan_set_access_mask(0); \
	429	kcsan_atomic_next(1); \
	430	} while (0)
	431
dfd402a4	432	#endif /* _LINUX_KCSAN_CHECKS_H */