locking/atomic: Correct (cmp)xchg() instrumentation

[linux-block.git] / include / linux / atomic / atomic-instrumented.h

diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h
index 7a139ec030b0c32bcdfc11df93ffb741c48d0282..03a232a1fa5782a668f77216b89fffc212b524a8 100644 (file)
--- a/include/linux/atomic/atomic-instrumented.h
+++ b/include/linux/atomic/atomic-instrumented.h
@@ -592,6 +592,28 @@ atomic_add_negative(int i, atomic_t *v)
        return arch_atomic_add_negative(i, v);
 }
 
+static __always_inline bool
+atomic_add_negative_acquire(int i, atomic_t *v)
+{
+       instrument_atomic_read_write(v, sizeof(*v));
+       return arch_atomic_add_negative_acquire(i, v);
+}
+
+static __always_inline bool
+atomic_add_negative_release(int i, atomic_t *v)
+{
+       kcsan_release();
+       instrument_atomic_read_write(v, sizeof(*v));
+       return arch_atomic_add_negative_release(i, v);
+}
+
+static __always_inline bool
+atomic_add_negative_relaxed(int i, atomic_t *v)
+{
+       instrument_atomic_read_write(v, sizeof(*v));
+       return arch_atomic_add_negative_relaxed(i, v);
+}
+
 static __always_inline int
 atomic_fetch_add_unless(atomic_t *v, int a, int u)
 {
@@ -1211,6 +1233,28 @@ atomic64_add_negative(s64 i, atomic64_t *v)
        return arch_atomic64_add_negative(i, v);
 }
 
+static __always_inline bool
+atomic64_add_negative_acquire(s64 i, atomic64_t *v)
+{
+       instrument_atomic_read_write(v, sizeof(*v));
+       return arch_atomic64_add_negative_acquire(i, v);
+}
+
+static __always_inline bool
+atomic64_add_negative_release(s64 i, atomic64_t *v)
+{
+       kcsan_release();
+       instrument_atomic_read_write(v, sizeof(*v));
+       return arch_atomic64_add_negative_release(i, v);
+}
+
+static __always_inline bool
+atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
+{
+       instrument_atomic_read_write(v, sizeof(*v));
+       return arch_atomic64_add_negative_relaxed(i, v);
+}
+
 static __always_inline s64
 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
 {
@@ -1830,6 +1874,28 @@ atomic_long_add_negative(long i, atomic_long_t *v)
        return arch_atomic_long_add_negative(i, v);
 }
 
+static __always_inline bool
+atomic_long_add_negative_acquire(long i, atomic_long_t *v)
+{
+       instrument_atomic_read_write(v, sizeof(*v));
+       return arch_atomic_long_add_negative_acquire(i, v);
+}
+
+static __always_inline bool
+atomic_long_add_negative_release(long i, atomic_long_t *v)
+{
+       kcsan_release();
+       instrument_atomic_read_write(v, sizeof(*v));
+       return arch_atomic_long_add_negative_release(i, v);
+}
+
+static __always_inline bool
+atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
+{
+       instrument_atomic_read_write(v, sizeof(*v));
+       return arch_atomic_long_add_negative_relaxed(i, v);
+}
+
 static __always_inline long
 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
 {
@@ -1882,14 +1948,14 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_xchg(__ai_ptr, __VA_ARGS__); \
 })
 
 #define xchg_acquire(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -1897,14 +1963,14 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_release(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_xchg_release(__ai_ptr, __VA_ARGS__); \
 })
 
 #define xchg_relaxed(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -1912,14 +1978,14 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
 })
 
 #define cmpxchg_acquire(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -1927,14 +1993,14 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_release(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
 })
 
 #define cmpxchg_relaxed(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -1942,14 +2008,14 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
 })
 
 #define cmpxchg64_acquire(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -1957,14 +2023,14 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_release(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
 })
 
 #define cmpxchg64_relaxed(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -1973,8 +2039,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -1982,8 +2048,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -1992,8 +2058,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_release(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2001,8 +2067,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2011,8 +2077,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2020,8 +2086,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2030,8 +2096,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_release(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
@@ -2039,22 +2105,22 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
-       instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
 })
 
 #define cmpxchg_local(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
 })
 
 #define cmpxchg64_local(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -2062,15 +2128,33 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
 })
 
+#define try_cmpxchg_local(ptr, oldp, ...) \
+({ \
+       typeof(ptr) __ai_ptr = (ptr); \
+       typeof(oldp) __ai_oldp = (oldp); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
+})
+
+#define try_cmpxchg64_local(ptr, oldp, ...) \
+({ \
+       typeof(ptr) __ai_ptr = (ptr); \
+       typeof(oldp) __ai_oldp = (oldp); \
+       instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
+       instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
+       arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
+})
+
 #define cmpxchg_double(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
-       instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
        arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
 })
 
@@ -2078,9 +2162,9 @@ atomic_long_dec_if_positive(atomic_long_t *v)
 #define cmpxchg_double_local(ptr, ...) \
 ({ \
        typeof(ptr) __ai_ptr = (ptr); \
-       instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
+       instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
        arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
 })
 
 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
-// 764f741eb77a7ad565dc8d99ce2837d5542e8aee
+// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e