kcsan: Add core memory barrier instrumentation functions

Add the core memory barrier instrumentation functions. These invalidate
the current in-flight reordered access based on the rules for the
respective barrier types and in-flight access type.

To obtain barrier instrumentation that can be disabled via __no_kcsan
with appropriate compiler-support (and not just with objtool help),
barrier instrumentation repurposes __atomic_signal_fence(), instead of
inserting explicit calls. Crucially, __atomic_signal_fence() normally
does not map to any real instructions, but is still intercepted by
fsanitize=thread. As a result, like any other instrumentation done by
the compiler, barrier instrumentation can be disabled with __no_kcsan.

Unfortunately Clang and GCC currently differ in their __no_kcsan aka
__no_sanitize_thread behaviour with respect to builtin atomics (and
__tsan_func_{entry,exit}) instrumentation. This is already reflected in
Kconfig.kcsan's dependencies for KCSAN_WEAK_MEMORY. A later change will
introduce support for newer versions of Clang that can implement
__no_kcsan to also remove the additional instrumentation introduced by
KCSAN_WEAK_MEMORY.

Signed-off-by: Marco Elver <elver@google.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/include/linux/kcsan-checks.h b/include/linux/kcsan-checks.h
index a1c6a89fde7103a725dd51831524f813641c7117..9d2c869167f2e6f95a261f1eb2353ba42db60346 100644 (file)
--- a/include/linux/kcsan-checks.h
+++ b/include/linux/kcsan-checks.h
@@ -36,6 +36,36 @@
  */
 void __kcsan_check_access(const volatile void *ptr, size_t size, int type);
 
+/*
+ * See definition of __tsan_atomic_signal_fence() in kernel/kcsan/core.c.
+ * Note: The mappings are arbitrary, and do not reflect any real mappings of C11
+ * memory orders to the LKMM memory orders and vice-versa!
+ */
+#define __KCSAN_BARRIER_TO_SIGNAL_FENCE_mb     __ATOMIC_SEQ_CST
+#define __KCSAN_BARRIER_TO_SIGNAL_FENCE_wmb    __ATOMIC_ACQ_REL
+#define __KCSAN_BARRIER_TO_SIGNAL_FENCE_rmb    __ATOMIC_ACQUIRE
+#define __KCSAN_BARRIER_TO_SIGNAL_FENCE_release        __ATOMIC_RELEASE
+
+/**
+ * __kcsan_mb - full memory barrier instrumentation
+ */
+void __kcsan_mb(void);
+
+/**
+ * __kcsan_wmb - write memory barrier instrumentation
+ */
+void __kcsan_wmb(void);
+
+/**
+ * __kcsan_rmb - read memory barrier instrumentation
+ */
+void __kcsan_rmb(void);
+
+/**
+ * __kcsan_release - release barrier instrumentation
+ */
+void __kcsan_release(void);
+
 /**
  * kcsan_disable_current - disable KCSAN for the current context
  *
@@ -159,6 +189,10 @@ void kcsan_end_scoped_access(struct kcsan_scoped_access *sa);
 static inline void __kcsan_check_access(const volatile void *ptr, size_t size,
                                        int type) { }
 
+static inline void __kcsan_mb(void)                    { }
+static inline void __kcsan_wmb(void)                   { }
+static inline void __kcsan_rmb(void)                   { }
+static inline void __kcsan_release(void)               { }
 static inline void kcsan_disable_current(void)         { }
 static inline void kcsan_enable_current(void)          { }
 static inline void kcsan_enable_current_nowarn(void)   { }
@@ -191,12 +225,45 @@ static inline void kcsan_end_scoped_access(struct kcsan_scoped_access *sa) { }
  */
 #define __kcsan_disable_current kcsan_disable_current
 #define __kcsan_enable_current kcsan_enable_current_nowarn
-#else
+#else /* __SANITIZE_THREAD__ */
 static inline void kcsan_check_access(const volatile void *ptr, size_t size,
                                      int type) { }
 static inline void __kcsan_enable_current(void)  { }
 static inline void __kcsan_disable_current(void) { }
-#endif
+#endif /* __SANITIZE_THREAD__ */
+
+#if defined(CONFIG_KCSAN_WEAK_MEMORY) && defined(__SANITIZE_THREAD__)
+/*
+ * Normal barrier instrumentation is not done via explicit calls, but by mapping
+ * to a repurposed __atomic_signal_fence(), which normally does not generate any
+ * real instructions, but is still intercepted by fsanitize=thread. This means,
+ * like any other compile-time instrumentation, barrier instrumentation can be
+ * disabled with the __no_kcsan function attribute.
+ *
+ * Also see definition of __tsan_atomic_signal_fence() in kernel/kcsan/core.c.
+ */
+#define __KCSAN_BARRIER_TO_SIGNAL_FENCE(name)                                  \
+       static __always_inline void kcsan_##name(void)                          \
+       {                                                                       \
+               barrier();                                                      \
+               __atomic_signal_fence(__KCSAN_BARRIER_TO_SIGNAL_FENCE_##name);  \
+               barrier();                                                      \
+       }
+__KCSAN_BARRIER_TO_SIGNAL_FENCE(mb)
+__KCSAN_BARRIER_TO_SIGNAL_FENCE(wmb)
+__KCSAN_BARRIER_TO_SIGNAL_FENCE(rmb)
+__KCSAN_BARRIER_TO_SIGNAL_FENCE(release)
+#elif defined(CONFIG_KCSAN_WEAK_MEMORY) && defined(__KCSAN_INSTRUMENT_BARRIERS__)
+#define kcsan_mb       __kcsan_mb
+#define kcsan_wmb      __kcsan_wmb
+#define kcsan_rmb      __kcsan_rmb
+#define kcsan_release  __kcsan_release
+#else /* CONFIG_KCSAN_WEAK_MEMORY && ... */
+static inline void kcsan_mb(void)              { }
+static inline void kcsan_wmb(void)             { }
+static inline void kcsan_rmb(void)             { }
+static inline void kcsan_release(void)         { }
+#endif /* CONFIG_KCSAN_WEAK_MEMORY && ... */
 
 /**
  * __kcsan_check_read - check regular read access for races

diff --git a/kernel/kcsan/core.c b/kernel/kcsan/core.c
index 481f8a5240898fc7f7de9d24f9210d3bdac54e2e..916060913966645073d613f6c0cf9e0080c9e968 100644 (file)
--- a/kernel/kcsan/core.c
+++ b/kernel/kcsan/core.c
@@ -939,6 +939,22 @@ void __kcsan_check_access(const volatile void *ptr, size_t size, int type)
 }
 EXPORT_SYMBOL(__kcsan_check_access);
 
+#define DEFINE_MEMORY_BARRIER(name, order_before_cond)                         \
+       void __kcsan_##name(void)                                               \
+       {                                                                       \
+               struct kcsan_scoped_access *sa = get_reorder_access(get_ctx()); \
+               if (!sa)                                                        \
+                       return;                                                 \
+               if (order_before_cond)                                          \
+                       sa->size = 0;                                           \
+       }                                                                       \
+       EXPORT_SYMBOL(__kcsan_##name)
+
+DEFINE_MEMORY_BARRIER(mb, true);
+DEFINE_MEMORY_BARRIER(wmb, sa->type & (KCSAN_ACCESS_WRITE | KCSAN_ACCESS_COMPOUND));
+DEFINE_MEMORY_BARRIER(rmb, !(sa->type & KCSAN_ACCESS_WRITE) || (sa->type & KCSAN_ACCESS_COMPOUND));
+DEFINE_MEMORY_BARRIER(release, true);
+
 /*
  * KCSAN uses the same instrumentation that is emitted by supported compilers
  * for ThreadSanitizer (TSAN).
@@ -1123,10 +1139,19 @@ EXPORT_SYMBOL(__tsan_init);
  * functions, whose job is to also execute the operation itself.
  */
 
+static __always_inline void kcsan_atomic_builtin_memorder(int memorder)
+{
+       if (memorder == __ATOMIC_RELEASE ||
+           memorder == __ATOMIC_SEQ_CST ||
+           memorder == __ATOMIC_ACQ_REL)
+               __kcsan_release();
+}
+
 #define DEFINE_TSAN_ATOMIC_LOAD_STORE(bits)                                                        \
        u##bits __tsan_atomic##bits##_load(const u##bits *ptr, int memorder);                      \
        u##bits __tsan_atomic##bits##_load(const u##bits *ptr, int memorder)                       \
        {                                                                                          \
+               kcsan_atomic_builtin_memorder(memorder);                                           \
                if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) {                                    \
                        check_access(ptr, bits / BITS_PER_BYTE, KCSAN_ACCESS_ATOMIC, _RET_IP_);    \
                }                                                                                  \
@@ -1136,6 +1161,7 @@ EXPORT_SYMBOL(__tsan_init);
        void __tsan_atomic##bits##_store(u##bits *ptr, u##bits v, int memorder);                   \
        void __tsan_atomic##bits##_store(u##bits *ptr, u##bits v, int memorder)                    \
        {                                                                                          \
+               kcsan_atomic_builtin_memorder(memorder);                                           \
                if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) {                                    \
                        check_access(ptr, bits / BITS_PER_BYTE,                                    \
                                     KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC, _RET_IP_);          \
@@ -1148,6 +1174,7 @@ EXPORT_SYMBOL(__tsan_init);
        u##bits __tsan_atomic##bits##_##op(u##bits *ptr, u##bits v, int memorder);                 \
        u##bits __tsan_atomic##bits##_##op(u##bits *ptr, u##bits v, int memorder)                  \
        {                                                                                          \
+               kcsan_atomic_builtin_memorder(memorder);                                           \
                if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) {                                    \
                        check_access(ptr, bits / BITS_PER_BYTE,                                    \
                                     KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE |                  \
@@ -1180,6 +1207,7 @@ EXPORT_SYMBOL(__tsan_init);
        int __tsan_atomic##bits##_compare_exchange_##strength(u##bits *ptr, u##bits *exp,          \
                                                              u##bits val, int mo, int fail_mo)    \
        {                                                                                          \
+               kcsan_atomic_builtin_memorder(mo);                                                 \
                if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) {                                    \
                        check_access(ptr, bits / BITS_PER_BYTE,                                    \
                                     KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE |                  \
@@ -1195,6 +1223,7 @@ EXPORT_SYMBOL(__tsan_init);
        u##bits __tsan_atomic##bits##_compare_exchange_val(u##bits *ptr, u##bits exp, u##bits val, \
                                                           int mo, int fail_mo)                    \
        {                                                                                          \
+               kcsan_atomic_builtin_memorder(mo);                                                 \
                if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) {                                    \
                        check_access(ptr, bits / BITS_PER_BYTE,                                    \
                                     KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE |                  \
@@ -1226,10 +1255,47 @@ DEFINE_TSAN_ATOMIC_OPS(64);
 void __tsan_atomic_thread_fence(int memorder);
 void __tsan_atomic_thread_fence(int memorder)
 {
+       kcsan_atomic_builtin_memorder(memorder);
        __atomic_thread_fence(memorder);
 }
 EXPORT_SYMBOL(__tsan_atomic_thread_fence);
 
+/*
+ * In instrumented files, we emit instrumentation for barriers by mapping the
+ * kernel barriers to an __atomic_signal_fence(), which is interpreted specially
+ * and otherwise has no relation to a real __atomic_signal_fence(). No known
+ * kernel code uses __atomic_signal_fence().
+ *
+ * Since fsanitize=thread instrumentation handles __atomic_signal_fence(), which
+ * are turned into calls to __tsan_atomic_signal_fence(), such instrumentation
+ * can be disabled via the __no_kcsan function attribute (vs. an explicit call
+ * which could not). When __no_kcsan is requested, __atomic_signal_fence()
+ * generates no code.
+ *
+ * Note: The result of using __atomic_signal_fence() with KCSAN enabled is
+ * potentially limiting the compiler's ability to reorder operations; however,
+ * if barriers were instrumented with explicit calls (without LTO), the compiler
+ * couldn't optimize much anyway. The result of a hypothetical architecture
+ * using __atomic_signal_fence() in normal code would be KCSAN false negatives.
+ */
 void __tsan_atomic_signal_fence(int memorder);
-void __tsan_atomic_signal_fence(int memorder) { }
+noinline void __tsan_atomic_signal_fence(int memorder)
+{
+       switch (memorder) {
+       case __KCSAN_BARRIER_TO_SIGNAL_FENCE_mb:
+               __kcsan_mb();
+               break;
+       case __KCSAN_BARRIER_TO_SIGNAL_FENCE_wmb:
+               __kcsan_wmb();
+               break;
+       case __KCSAN_BARRIER_TO_SIGNAL_FENCE_rmb:
+               __kcsan_rmb();
+               break;
+       case __KCSAN_BARRIER_TO_SIGNAL_FENCE_release:
+               __kcsan_release();
+               break;
+       default:
+               break;
+       }
+}
 EXPORT_SYMBOL(__tsan_atomic_signal_fence);