membarrier: riscv: Add full memory barrier in switch_mm()

The membarrier system call requires a full memory barrier after storing
to rq->curr, before going back to user-space.  The barrier is only
needed when switching between processes: the barrier is implied by
mmdrop() when switching from kernel to userspace, and it's not needed
when switching from userspace to kernel.

Rely on the feature/mechanism ARCH_HAS_MEMBARRIER_CALLBACKS and on the
primitive membarrier_arch_switch_mm(), already adopted by the PowerPC
architecture, to insert the required barrier.

Fixes: fab957c11efe2f ("RISC-V: Atomic and Locking Code")
Signed-off-by: Andrea Parri <parri.andrea@gmail.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://lore.kernel.org/r/20240131144936.29190-2-parri.andrea@gmail.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/MAINTAINERS b/MAINTAINERS
index 8d1052fa6a6924d17a4d2681fa7907c544e35186..2450e88d3e2c2c38483b795e7a586f07eee4d010 100644 (file)
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -14039,7 +14039,7 @@ M:      Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 M:     "Paul E. McKenney" <paulmck@kernel.org>
 L:     linux-kernel@vger.kernel.org
 S:     Supported
-F:     arch/powerpc/include/asm/membarrier.h
+F:     arch/*/include/asm/membarrier.h
 F:     include/uapi/linux/membarrier.h
 F:     kernel/sched/membarrier.c
 

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index bffbd869a0682842883591788da784648acf1626..087abf9e51c69015dd647be917c108f48be55d55 100644 (file)
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -27,6 +27,7 @@ config RISCV
        select ARCH_HAS_GCOV_PROFILE_ALL
        select ARCH_HAS_GIGANTIC_PAGE
        select ARCH_HAS_KCOV
+       select ARCH_HAS_MEMBARRIER_CALLBACKS
        select ARCH_HAS_MMIOWB
        select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
        select ARCH_HAS_PMEM_API

diff --git a/arch/riscv/include/asm/membarrier.h b/arch/riscv/include/asm/membarrier.h
new file mode 100644 (file)
index 0000000..6c016eb
--- /dev/null
+++ b/arch/riscv/include/asm/membarrier.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _ASM_RISCV_MEMBARRIER_H
+#define _ASM_RISCV_MEMBARRIER_H
+
+static inline void membarrier_arch_switch_mm(struct mm_struct *prev,
+                                            struct mm_struct *next,
+                                            struct task_struct *tsk)
+{
+       /*
+        * Only need the full barrier when switching between processes.
+        * Barrier when switching from kernel to userspace is not
+        * required here, given that it is implied by mmdrop(). Barrier
+        * when switching from userspace to kernel is not needed after
+        * store to rq->curr.
+        */
+       if (IS_ENABLED(CONFIG_SMP) &&
+           likely(!(atomic_read(&next->membarrier_state) &
+                    (MEMBARRIER_STATE_PRIVATE_EXPEDITED |
+                     MEMBARRIER_STATE_GLOBAL_EXPEDITED)) || !prev))
+               return;
+
+       /*
+        * The membarrier system call requires a full memory barrier
+        * after storing to rq->curr, before going back to user-space.
+        * Matches a full barrier in the proximity of the membarrier
+        * system call entry.
+        */
+       smp_mb();
+}
+
+#endif /* _ASM_RISCV_MEMBARRIER_H */

diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c
index 217fd4de6134224655db81759f0b35c73948e46d..ba8eb3944687cfd445770c357b32cb4fa0e37564 100644 (file)
--- a/arch/riscv/mm/context.c
+++ b/arch/riscv/mm/context.c
@@ -323,6 +323,8 @@ void switch_mm(struct mm_struct *prev, struct mm_struct *next,
        if (unlikely(prev == next))
                return;
 
+       membarrier_arch_switch_mm(prev, next, task);
+
        /*
         * Mark the current MM context as inactive, and the next as
         * active.  This is at least used by the icache flushing

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9116bcc903467fe0d5854e3deb06b8d334cf85eb..c4ca8085885a3a345d63cd04c7351e00a2248dd3 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6709,8 +6709,9 @@ static void __sched notrace __schedule(unsigned int sched_mode)
                 *
                 * Here are the schemes providing that barrier on the
                 * various architectures:
-                * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC.
-                *   switch_mm() rely on membarrier_arch_switch_mm() on PowerPC.
+                * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC,
+                *   RISC-V.  switch_mm() relies on membarrier_arch_switch_mm()
+                *   on PowerPC and on RISC-V.
                 * - finish_lock_switch() for weakly-ordered
                 *   architectures where spin_unlock is a full barrier,
                 * - switch_to() for arm64 (weakly-ordered, spin_unlock