Merge tag 'kthread-cleanups-for-v5.19' of git://git.kernel.org/pub/scm/linux/kernel...

[linux-block.git] / arch / xtensa / kernel / process.c

diff --git a/arch/xtensa/kernel/process.c b/arch/xtensa/kernel/process.c
index c3751cc88e5dc94378bd58d639331152390b9395..68e0e2f06d660a9b411ca81a68554c73a33d5f5a 100644 (file)
--- a/arch/xtensa/kernel/process.c
+++ b/arch/xtensa/kernel/process.c
@@ -47,6 +47,7 @@
 #include <asm/asm-offsets.h>
 #include <asm/regs.h>
 #include <asm/hw_breakpoint.h>
+#include <asm/traps.h>
 
 extern void ret_from_fork(void);
 extern void ret_from_kernel_thread(void);
@@ -63,52 +64,114 @@ EXPORT_SYMBOL(__stack_chk_guard);
 
 #if XTENSA_HAVE_COPROCESSORS
 
-void coprocessor_release_all(struct thread_info *ti)
+void local_coprocessors_flush_release_all(void)
 {
-       unsigned long cpenable;
-       int i;
+       struct thread_info **coprocessor_owner;
+       struct thread_info *unique_owner[XCHAL_CP_MAX];
+       int n = 0;
+       int i, j;
 
-       /* Make sure we don't switch tasks during this operation. */
+       coprocessor_owner = this_cpu_ptr(&exc_table)->coprocessor_owner;
+       xtensa_set_sr(XCHAL_CP_MASK, cpenable);
 
-       preempt_disable();
+       for (i = 0; i < XCHAL_CP_MAX; i++) {
+               struct thread_info *ti = coprocessor_owner[i];
 
-       /* Walk through all cp owners and release it for the requested one. */
+               if (ti) {
+                       coprocessor_flush(ti, i);
 
-       cpenable = ti->cpenable;
+                       for (j = 0; j < n; j++)
+                               if (unique_owner[j] == ti)
+                                       break;
+                       if (j == n)
+                               unique_owner[n++] = ti;
 
-       for (i = 0; i < XCHAL_CP_MAX; i++) {
-               if (coprocessor_owner[i] == ti) {
-                       coprocessor_owner[i] = 0;
-                       cpenable &= ~(1 << i);
+                       coprocessor_owner[i] = NULL;
                }
        }
+       for (i = 0; i < n; i++) {
+               /* pairs with memw (1) in fast_coprocessor and memw in switch_to */
+               smp_wmb();
+               unique_owner[i]->cpenable = 0;
+       }
+       xtensa_set_sr(0, cpenable);
+}
 
-       ti->cpenable = cpenable;
+static void local_coprocessor_release_all(void *info)
+{
+       struct thread_info *ti = info;
+       struct thread_info **coprocessor_owner;
+       int i;
+
+       coprocessor_owner = this_cpu_ptr(&exc_table)->coprocessor_owner;
+
+       /* Walk through all cp owners and release it for the requested one. */
+
+       for (i = 0; i < XCHAL_CP_MAX; i++) {
+               if (coprocessor_owner[i] == ti)
+                       coprocessor_owner[i] = NULL;
+       }
+       /* pairs with memw (1) in fast_coprocessor and memw in switch_to */
+       smp_wmb();
+       ti->cpenable = 0;
        if (ti == current_thread_info())
                xtensa_set_sr(0, cpenable);
+}
 
-       preempt_enable();
+void coprocessor_release_all(struct thread_info *ti)
+{
+       if (ti->cpenable) {
+               /* pairs with memw (2) in fast_coprocessor */
+               smp_rmb();
+               smp_call_function_single(ti->cp_owner_cpu,
+                                        local_coprocessor_release_all,
+                                        ti, true);
+       }
 }
 
-void coprocessor_flush_all(struct thread_info *ti)
+static void local_coprocessor_flush_all(void *info)
 {
-       unsigned long cpenable, old_cpenable;
+       struct thread_info *ti = info;
+       struct thread_info **coprocessor_owner;
+       unsigned long old_cpenable;
        int i;
 
-       preempt_disable();
-
-       old_cpenable = xtensa_get_sr(cpenable);
-       cpenable = ti->cpenable;
-       xtensa_set_sr(cpenable, cpenable);
+       coprocessor_owner = this_cpu_ptr(&exc_table)->coprocessor_owner;
+       old_cpenable = xtensa_xsr(ti->cpenable, cpenable);
 
        for (i = 0; i < XCHAL_CP_MAX; i++) {
-               if ((cpenable & 1) != 0 && coprocessor_owner[i] == ti)
+               if (coprocessor_owner[i] == ti)
                        coprocessor_flush(ti, i);
-               cpenable >>= 1;
        }
        xtensa_set_sr(old_cpenable, cpenable);
+}
 
-       preempt_enable();
+void coprocessor_flush_all(struct thread_info *ti)
+{
+       if (ti->cpenable) {
+               /* pairs with memw (2) in fast_coprocessor */
+               smp_rmb();
+               smp_call_function_single(ti->cp_owner_cpu,
+                                        local_coprocessor_flush_all,
+                                        ti, true);
+       }
+}
+
+static void local_coprocessor_flush_release_all(void *info)
+{
+       local_coprocessor_flush_all(info);
+       local_coprocessor_release_all(info);
+}
+
+void coprocessor_flush_release_all(struct thread_info *ti)
+{
+       if (ti->cpenable) {
+               /* pairs with memw (2) in fast_coprocessor */
+               smp_rmb();
+               smp_call_function_single(ti->cp_owner_cpu,
+                                        local_coprocessor_flush_release_all,
+                                        ti, true);
+       }
 }
 
 #endif
@@ -140,8 +203,7 @@ void flush_thread(void)
 {
 #if XTENSA_HAVE_COPROCESSORS
        struct thread_info *ti = current_thread_info();
-       coprocessor_flush_all(ti);
-       coprocessor_release_all(ti);
+       coprocessor_flush_release_all(ti);
 #endif
        flush_ptrace_hw_breakpoint(current);
 }