locking/atomic: Correct (cmp)xchg() instrumentation

[linux-block.git] / include / linux / atomic / atomic-instrumented.h

diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h
index 245ba661c49389723a0dcc568969ce899b82af60..03a232a1fa5782a668f77216b89fffc212b524a8 100644 (file)
--- a/include/linux/atomic/atomic-instrumented.h
+++ b/include/linux/atomic/atomic-instrumented.h
@@ -1948,14 +1948,14 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_xchg(__ai_ptr, __VA_ARGS__); \
 })
 
 #define xchg_acquire(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -1963,14 +1963,14 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_release(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_xchg_release(__ai_ptr, __VA_ARGS__); \
 })
 
 #define xchg_relaxed(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -1978,14 +1978,14 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
 })
 
 #define cmpxchg_acquire(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -1993,14 +1993,14 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_release(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
 })
 
 #define cmpxchg_relaxed(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -2008,14 +2008,14 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
 })
 
 #define cmpxchg64_acquire(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -2023,14 +2023,14 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_release(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
 })
 
 #define cmpxchg64_relaxed(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -2039,8 +2039,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2048,8 +2048,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2058,8 +2058,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_release(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2067,8 +2067,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2077,8 +2077,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2086,8 +2086,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2096,8 +2096,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_release(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2105,22 +2105,22 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
 #define cmpxchg_local(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
 })
 
 #define cmpxchg64_local(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -2128,7 +2128,7 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -2136,8 +2136,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2145,8 +2145,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2154,7 +2154,7 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
        arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -2162,9 +2162,9 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 #define cmpxchg_double_local(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
        arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
 })
 
 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
-// 97fe4d79aa058d2164df824632cbc4f716d2a407
+// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e