six locks: Fix a lost wakeup

There was a lost wakeup between a read unlock in percpu mode and a write
lock. The unlock path unlocks, then executes a barrier, then checks for
waiters; correspondingly, the lock side should set the wait bit and
execute a barrier, then attempt to take the lock.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/six.c b/fs/bcachefs/six.c
index 01ff210ff18c70f9c2ebbdbb42cd1212cf34ee99..abdc2414f58b15dd62e837ad84e6ee86a6fe0555 100644 (file)
--- a/fs/bcachefs/six.c
+++ b/fs/bcachefs/six.c
@@ -198,6 +198,14 @@ retry:
                        atomic64_add(__SIX_VAL(write_locking, 1),
                                     &lock->state.counter);
                        smp_mb__after_atomic();
+               } else if (!(lock->state.waiters & (1 << SIX_LOCK_write))) {
+                       atomic64_add(__SIX_VAL(waiters, 1 << SIX_LOCK_write),
+                                    &lock->state.counter);
+                       /*
+                        * pairs with barrier after unlock and before checking
+                        * for readers in unlock path
+                        */
+                       smp_mb__after_atomic();
                }
 
                ret = !pcpu_read_count(lock);
@@ -212,9 +220,6 @@ retry:
                if (ret || try)
                        v -= __SIX_VAL(write_locking, 1);
 
-               if (!ret && !try && !(lock->state.waiters & (1 << SIX_LOCK_write)))
-                       v += __SIX_VAL(waiters, 1 << SIX_LOCK_write);
-
                if (try && !ret) {
                        old.v = atomic64_add_return(v, &lock->state.counter);
                        six_lock_wakeup(lock, old, SIX_LOCK_read);