riscv: Add qspinlock support

In order to produce a generic kernel, a user can select
CONFIG_COMBO_SPINLOCKS which will fallback at runtime to the ticket
spinlock implementation if Zabha or Ziccrse are not present.

Note that we can't use alternatives here because the discovery of
extensions is done too late and we need to start with the qspinlock
implementation because the ticket spinlock implementation would pollute
the spinlock value, so let's use static keys.

This is largely based on Guo's work and Leonardo reviews at [1].

Link: https://lore.kernel.org/linux-riscv/20231225125847.2778638-1-guoren@kernel.org/
Signed-off-by: Guo Ren <guoren@kernel.org>
Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com>
Reviewed-by: Andrea Parri <parri.andrea@gmail.com>
Link: https://lore.kernel.org/r/20241103145153.105097-14-alexghiti@rivosinc.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/Documentation/features/locking/queued-spinlocks/arch-support.txt b/Documentation/features/locking/queued-spinlocks/arch-support.txt
index 22f2990392ff8f5cb758d156609926d080eb945a..cf26042480e21d1fed2a0626c70c294482d9000b 100644 (file)
--- a/Documentation/features/locking/queued-spinlocks/arch-support.txt
+++ b/Documentation/features/locking/queued-spinlocks/arch-support.txt
@@ -20,7 +20,7 @@
     |    openrisc: |  ok  |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
-    |       riscv: | TODO |
+    |       riscv: |  ok  |
     |        s390: | TODO |
     |          sh: | TODO |
     |       sparc: |  ok  |

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 093ee6537331205353f7fad97652efc158269b6d..f5698ecc5cccddb3e900cc70d841596cfbd03fca 100644 (file)
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -82,6 +82,7 @@ config RISCV
        select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
        select ARCH_WANTS_NO_INSTR
        select ARCH_WANTS_THP_SWAP if HAVE_ARCH_TRANSPARENT_HUGEPAGE
+       select ARCH_WEAK_RELEASE_ACQUIRE if ARCH_USE_QUEUED_SPINLOCKS
        select BINFMT_FLAT_NO_DATA_START_OFFSET if !MMU
        select BUILDTIME_TABLE_SORT if MMU
        select CLINT_TIMER if RISCV_M_MODE
@@ -507,6 +508,39 @@ config NODES_SHIFT
          Specify the maximum number of NUMA Nodes available on the target
          system.  Increases memory reserved to accommodate various tables.
 
+choice
+       prompt "RISC-V spinlock type"
+       default RISCV_COMBO_SPINLOCKS
+
+config RISCV_TICKET_SPINLOCKS
+       bool "Using ticket spinlock"
+
+config RISCV_QUEUED_SPINLOCKS
+       bool "Using queued spinlock"
+       depends on SMP && MMU && NONPORTABLE
+       select ARCH_USE_QUEUED_SPINLOCKS
+       help
+         The queued spinlock implementation requires the forward progress
+         guarantee of cmpxchg()/xchg() atomic operations: CAS with Zabha or
+         LR/SC with Ziccrse provide such guarantee.
+
+         Select this if and only if Zabha or Ziccrse is available on your
+         platform, RISCV_QUEUED_SPINLOCKS must not be selected for platforms
+         without one of those extensions.
+
+         If unsure, select RISCV_COMBO_SPINLOCKS, which will use qspinlocks
+         when supported and otherwise ticket spinlocks.
+
+config RISCV_COMBO_SPINLOCKS
+       bool "Using combo spinlock"
+       depends on SMP && MMU
+       select ARCH_USE_QUEUED_SPINLOCKS
+       help
+         Embed both queued spinlock and ticket lock so that the spinlock
+         implementation can be chosen at runtime.
+
+endchoice
+
 config RISCV_ALTERNATIVE
        bool
        depends on !XIP_KERNEL

diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild
index 1461af12da6e2bbbff6cf737a7babf33bd298cdd..de13d5a234f882cfefd06d0939d36762b02dbaf9 100644 (file)
--- a/arch/riscv/include/asm/Kbuild
+++ b/arch/riscv/include/asm/Kbuild
@@ -6,10 +6,12 @@ generic-y += early_ioremap.h
 generic-y += flat.h
 generic-y += kvm_para.h
 generic-y += mmzone.h
+generic-y += mcs_spinlock.h
 generic-y += parport.h
-generic-y += spinlock.h
 generic-y += spinlock_types.h
+generic-y += ticket_spinlock.h
 generic-y += qrwlock.h
 generic-y += qrwlock_types.h
+generic-y += qspinlock.h
 generic-y += user.h
 generic-y += vmlinux.lds.h

diff --git a/arch/riscv/include/asm/spinlock.h b/arch/riscv/include/asm/spinlock.h
new file mode 100644 (file)
index 0000000..e5121b8
--- /dev/null
+++ b/arch/riscv/include/asm/spinlock.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __ASM_RISCV_SPINLOCK_H
+#define __ASM_RISCV_SPINLOCK_H
+
+#ifdef CONFIG_RISCV_COMBO_SPINLOCKS
+#define _Q_PENDING_LOOPS       (1 << 9)
+
+#define __no_arch_spinlock_redefine
+#include <asm/ticket_spinlock.h>
+#include <asm/qspinlock.h>
+#include <asm/jump_label.h>
+
+/*
+ * TODO: Use an alternative instead of a static key when we are able to parse
+ * the extensions string earlier in the boot process.
+ */
+DECLARE_STATIC_KEY_TRUE(qspinlock_key);
+
+#define SPINLOCK_BASE_DECLARE(op, type, type_lock)                     \
+static __always_inline type arch_spin_##op(type_lock lock)             \
+{                                                                      \
+       if (static_branch_unlikely(&qspinlock_key))                     \
+               return queued_spin_##op(lock);                          \
+       return ticket_spin_##op(lock);                                  \
+}
+
+SPINLOCK_BASE_DECLARE(lock, void, arch_spinlock_t *)
+SPINLOCK_BASE_DECLARE(unlock, void, arch_spinlock_t *)
+SPINLOCK_BASE_DECLARE(is_locked, int, arch_spinlock_t *)
+SPINLOCK_BASE_DECLARE(is_contended, int, arch_spinlock_t *)
+SPINLOCK_BASE_DECLARE(trylock, bool, arch_spinlock_t *)
+SPINLOCK_BASE_DECLARE(value_unlocked, int, arch_spinlock_t)
+
+#elif defined(CONFIG_RISCV_QUEUED_SPINLOCKS)
+
+#include <asm/qspinlock.h>
+
+#else
+
+#include <asm/ticket_spinlock.h>
+
+#endif
+
+#include <asm/qrwlock.h>
+
+#endif /* __ASM_RISCV_SPINLOCK_H */

diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index a2cde65b69e950d929e3f9a5165d5f72d307e850..438e4f6ad2adeb5d2a924d90d0af0c5ded92bda3 100644 (file)
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -244,6 +244,42 @@ static void __init parse_dtb(void)
 #endif
 }
 
+#if defined(CONFIG_RISCV_COMBO_SPINLOCKS)
+DEFINE_STATIC_KEY_TRUE(qspinlock_key);
+EXPORT_SYMBOL(qspinlock_key);
+#endif
+
+static void __init riscv_spinlock_init(void)
+{
+       char *using_ext = NULL;
+
+       if (IS_ENABLED(CONFIG_RISCV_TICKET_SPINLOCKS)) {
+               pr_info("Ticket spinlock: enabled\n");
+               return;
+       }
+
+       if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) &&
+           IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) &&
+           riscv_isa_extension_available(NULL, ZABHA) &&
+           riscv_isa_extension_available(NULL, ZACAS)) {
+               using_ext = "using Zabha";
+       } else if (riscv_isa_extension_available(NULL, ZICCRSE)) {
+               using_ext = "using Ziccrse";
+       }
+#if defined(CONFIG_RISCV_COMBO_SPINLOCKS)
+       else {
+               static_branch_disable(&qspinlock_key);
+               pr_info("Ticket spinlock: enabled\n");
+               return;
+       }
+#endif
+
+       if (!using_ext)
+               pr_err("Queued spinlock without Zabha or Ziccrse");
+       else
+               pr_info("Queued spinlock %s: enabled\n", using_ext);
+}
+
 extern void __init init_rt_signal_env(void);
 
 void __init setup_arch(char **cmdline_p)
@@ -297,6 +333,7 @@ void __init setup_arch(char **cmdline_p)
        riscv_set_dma_cache_alignment();
 
        riscv_user_isa_enable();
+       riscv_spinlock_init();
 }
 
 bool arch_cpu_is_hotpluggable(int cpu)

diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h
index 0655aa5b57b29066286b9c94f477e7d6593f88b8..bf47cca2c375dca436c31d8ce7df481755247626 100644 (file)
--- a/include/asm-generic/qspinlock.h
+++ b/include/asm-generic/qspinlock.h
@@ -136,6 +136,7 @@ static __always_inline bool virt_spin_lock(struct qspinlock *lock)
 }
 #endif
 
+#ifndef __no_arch_spinlock_redefine
 /*
  * Remapping spinlock architecture specific functions to the corresponding
  * queued spinlock functions.
@@ -146,5 +147,6 @@ static __always_inline bool virt_spin_lock(struct qspinlock *lock)
 #define arch_spin_lock(l)              queued_spin_lock(l)
 #define arch_spin_trylock(l)           queued_spin_trylock(l)
 #define arch_spin_unlock(l)            queued_spin_unlock(l)
+#endif
 
 #endif /* __ASM_GENERIC_QSPINLOCK_H */

diff --git a/include/asm-generic/ticket_spinlock.h b/include/asm-generic/ticket_spinlock.h
index cfcff22b37b390f70af96f985392f97196c877e7..325779970d8a051a5571acd92a4c14cd49f7e52e 100644 (file)
--- a/include/asm-generic/ticket_spinlock.h
+++ b/include/asm-generic/ticket_spinlock.h
@@ -89,6 +89,7 @@ static __always_inline int ticket_spin_is_contended(arch_spinlock_t *lock)
        return (s16)((val >> 16) - (val & 0xffff)) > 1;
 }
 
+#ifndef __no_arch_spinlock_redefine
 /*
  * Remapping spinlock architecture specific functions to the corresponding
  * ticket spinlock functions.
@@ -99,5 +100,6 @@ static __always_inline int ticket_spin_is_contended(arch_spinlock_t *lock)
 #define arch_spin_lock(l)              ticket_spin_lock(l)
 #define arch_spin_trylock(l)           ticket_spin_trylock(l)
 #define arch_spin_unlock(l)            ticket_spin_unlock(l)
+#endif
 
 #endif /* __ASM_GENERIC_TICKET_SPINLOCK_H */