Merge branch 'locking-rwsem-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
authorLinus Torvalds <torvalds@linux-foundation.org>
Mon, 16 May 2016 20:41:02 +0000 (13:41 -0700)
committerLinus Torvalds <torvalds@linux-foundation.org>
Mon, 16 May 2016 20:41:02 +0000 (13:41 -0700)
Pull support for killable rwsems from Ingo Molnar:
 "This, by Michal Hocko, implements down_write_killable().

  The main usecase will be to update mm_sem usage sites to use this new
  API, to allow the mm-reaper introduced in commit aac453635549 ("mm,
  oom: introduce oom reaper") to tear down oom victim address spaces
  asynchronously with minimum latencies and without deadlock worries"

[ The vfs will want it too as the inode lock is changed from a mutex to
  a rwsem due to the parallel lookup and readdir updates ]

* 'locking-rwsem-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  locking/rwsem: Fix comment on register clobbering
  locking/rwsem: Fix down_write_killable()
  locking/rwsem, x86: Add frame annotation for call_rwsem_down_write_failed_killable()
  locking/rwsem: Provide down_write_killable()
  locking/rwsem, x86: Provide __down_write_killable()
  locking/rwsem, s390: Provide __down_write_killable()
  locking/rwsem, ia64: Provide __down_write_killable()
  locking/rwsem, alpha: Provide __down_write_killable()
  locking/rwsem: Introduce basis for down_write_killable()
  locking/rwsem, sparc: Drop superfluous arch specific implementation
  locking/rwsem, sh: Drop superfluous arch specific implementation
  locking/rwsem, xtensa: Drop superfluous arch specific implementation
  locking/rwsem: Drop explicit memory barriers
  locking/rwsem: Get rid of __down_write_nested()

18 files changed:
arch/alpha/include/asm/rwsem.h
arch/ia64/include/asm/rwsem.h
arch/s390/include/asm/rwsem.h
arch/sh/include/asm/Kbuild
arch/sh/include/asm/rwsem.h [deleted file]
arch/sparc/include/asm/Kbuild
arch/sparc/include/asm/rwsem.h [deleted file]
arch/x86/include/asm/rwsem.h
arch/x86/lib/rwsem.S
arch/xtensa/include/asm/Kbuild
arch/xtensa/include/asm/rwsem.h [deleted file]
include/asm-generic/rwsem.h
include/linux/lockdep.h
include/linux/rwsem-spinlock.h
include/linux/rwsem.h
kernel/locking/rwsem-spinlock.c
kernel/locking/rwsem-xadd.c
kernel/locking/rwsem.c

index a83bbea62c67481442cb0bd04ba033b7ef95b848..0131a7058778ed67fd32da6595052a0287b742fa 100644 (file)
@@ -63,7 +63,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
        return res >= 0 ? 1 : 0;
 }
 
-static inline void __down_write(struct rw_semaphore *sem)
+static inline long ___down_write(struct rw_semaphore *sem)
 {
        long oldcount;
 #ifndef        CONFIG_SMP
@@ -83,10 +83,24 @@ static inline void __down_write(struct rw_semaphore *sem)
        :"=&r" (oldcount), "=m" (sem->count), "=&r" (temp)
        :"Ir" (RWSEM_ACTIVE_WRITE_BIAS), "m" (sem->count) : "memory");
 #endif
-       if (unlikely(oldcount))
+       return oldcount;
+}
+
+static inline void __down_write(struct rw_semaphore *sem)
+{
+       if (unlikely(___down_write(sem)))
                rwsem_down_write_failed(sem);
 }
 
+static inline int __down_write_killable(struct rw_semaphore *sem)
+{
+       if (unlikely(___down_write(sem)))
+               if (IS_ERR(rwsem_down_write_failed_killable(sem)))
+                       return -EINTR;
+
+       return 0;
+}
+
 /*
  * trylock for writing -- returns 1 if successful, 0 if contention
  */
index ce112472bdd65a10072eecf995ef5053cbae2e41..8b23e070b8440e0ce0989ec07bb416c55ad86e35 100644 (file)
@@ -49,8 +49,8 @@ __down_read (struct rw_semaphore *sem)
 /*
  * lock for writing
  */
-static inline void
-__down_write (struct rw_semaphore *sem)
+static inline long
+___down_write (struct rw_semaphore *sem)
 {
        long old, new;
 
@@ -59,10 +59,26 @@ __down_write (struct rw_semaphore *sem)
                new = old + RWSEM_ACTIVE_WRITE_BIAS;
        } while (cmpxchg_acq(&sem->count, old, new) != old);
 
-       if (old != 0)
+       return old;
+}
+
+static inline void
+__down_write (struct rw_semaphore *sem)
+{
+       if (___down_write(sem))
                rwsem_down_write_failed(sem);
 }
 
+static inline int
+__down_write_killable (struct rw_semaphore *sem)
+{
+       if (___down_write(sem))
+               if (IS_ERR(rwsem_down_write_failed_killable(sem)))
+                       return -EINTR;
+
+       return 0;
+}
+
 /*
  * unlock after reading
  */
index fead491dfc28522017b1be01b1160ce788d052cb..c75e4471e618826a385c6aef2955b79c757ab9e7 100644 (file)
@@ -90,7 +90,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
 /*
  * lock for writing
  */
-static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
+static inline long ___down_write(struct rw_semaphore *sem)
 {
        signed long old, new, tmp;
 
@@ -104,13 +104,23 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
                : "=&d" (old), "=&d" (new), "=Q" (sem->count)
                : "Q" (sem->count), "m" (tmp)
                : "cc", "memory");
-       if (old != 0)
-               rwsem_down_write_failed(sem);
+
+       return old;
 }
 
 static inline void __down_write(struct rw_semaphore *sem)
 {
-       __down_write_nested(sem, 0);
+       if (___down_write(sem))
+               rwsem_down_write_failed(sem);
+}
+
+static inline int __down_write_killable(struct rw_semaphore *sem)
+{
+       if (___down_write(sem))
+               if (IS_ERR(rwsem_down_write_failed_killable(sem)))
+                       return -EINTR;
+
+       return 0;
 }
 
 /*
index a319745a7b635e16561fe33a061486372abf2004..751c3373a92c8882331fa8dfa6e567c4c4be7584 100644 (file)
@@ -26,6 +26,7 @@ generic-y += percpu.h
 generic-y += poll.h
 generic-y += preempt.h
 generic-y += resource.h
+generic-y += rwsem.h
 generic-y += sembuf.h
 generic-y += serial.h
 generic-y += shmbuf.h
diff --git a/arch/sh/include/asm/rwsem.h b/arch/sh/include/asm/rwsem.h
deleted file mode 100644 (file)
index edab572..0000000
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * include/asm-sh/rwsem.h: R/W semaphores for SH using the stuff
- * in lib/rwsem.c.
- */
-
-#ifndef _ASM_SH_RWSEM_H
-#define _ASM_SH_RWSEM_H
-
-#ifndef _LINUX_RWSEM_H
-#error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead"
-#endif
-
-#ifdef __KERNEL__
-
-#define RWSEM_UNLOCKED_VALUE           0x00000000
-#define RWSEM_ACTIVE_BIAS              0x00000001
-#define RWSEM_ACTIVE_MASK              0x0000ffff
-#define RWSEM_WAITING_BIAS             (-0x00010000)
-#define RWSEM_ACTIVE_READ_BIAS         RWSEM_ACTIVE_BIAS
-#define RWSEM_ACTIVE_WRITE_BIAS                (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
-
-/*
- * lock for reading
- */
-static inline void __down_read(struct rw_semaphore *sem)
-{
-       if (atomic_inc_return((atomic_t *)(&sem->count)) > 0)
-               smp_wmb();
-       else
-               rwsem_down_read_failed(sem);
-}
-
-static inline int __down_read_trylock(struct rw_semaphore *sem)
-{
-       int tmp;
-
-       while ((tmp = sem->count) >= 0) {
-               if (tmp == cmpxchg(&sem->count, tmp,
-                                  tmp + RWSEM_ACTIVE_READ_BIAS)) {
-                       smp_wmb();
-                       return 1;
-               }
-       }
-       return 0;
-}
-
-/*
- * lock for writing
- */
-static inline void __down_write(struct rw_semaphore *sem)
-{
-       int tmp;
-
-       tmp = atomic_add_return(RWSEM_ACTIVE_WRITE_BIAS,
-                               (atomic_t *)(&sem->count));
-       if (tmp == RWSEM_ACTIVE_WRITE_BIAS)
-               smp_wmb();
-       else
-               rwsem_down_write_failed(sem);
-}
-
-static inline int __down_write_trylock(struct rw_semaphore *sem)
-{
-       int tmp;
-
-       tmp = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
-                     RWSEM_ACTIVE_WRITE_BIAS);
-       smp_wmb();
-       return tmp == RWSEM_UNLOCKED_VALUE;
-}
-
-/*
- * unlock after reading
- */
-static inline void __up_read(struct rw_semaphore *sem)
-{
-       int tmp;
-
-       smp_wmb();
-       tmp = atomic_dec_return((atomic_t *)(&sem->count));
-       if (tmp < -1 && (tmp & RWSEM_ACTIVE_MASK) == 0)
-               rwsem_wake(sem);
-}
-
-/*
- * unlock after writing
- */
-static inline void __up_write(struct rw_semaphore *sem)
-{
-       smp_wmb();
-       if (atomic_sub_return(RWSEM_ACTIVE_WRITE_BIAS,
-                             (atomic_t *)(&sem->count)) < 0)
-               rwsem_wake(sem);
-}
-
-/*
- * implement atomic add functionality
- */
-static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
-{
-       atomic_add(delta, (atomic_t *)(&sem->count));
-}
-
-/*
- * downgrade write lock to read lock
- */
-static inline void __downgrade_write(struct rw_semaphore *sem)
-{
-       int tmp;
-
-       smp_wmb();
-       tmp = atomic_add_return(-RWSEM_WAITING_BIAS, (atomic_t *)(&sem->count));
-       if (tmp < 0)
-               rwsem_downgrade_wake(sem);
-}
-
-static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
-{
-       __down_write(sem);
-}
-
-/*
- * implement exchange and add functionality
- */
-static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)
-{
-       smp_mb();
-       return atomic_add_return(delta, (atomic_t *)(&sem->count));
-}
-
-#endif /* __KERNEL__ */
-#endif /* _ASM_SH_RWSEM_H */
index e928618838bc53c83555cc7b275073fc8c745b3d..6024c26c058565c76f20ebe98a3c3b0ad5a96120 100644 (file)
@@ -16,6 +16,7 @@ generic-y += mm-arch-hooks.h
 generic-y += module.h
 generic-y += mutex.h
 generic-y += preempt.h
+generic-y += rwsem.h
 generic-y += serial.h
 generic-y += trace_clock.h
 generic-y += types.h
diff --git a/arch/sparc/include/asm/rwsem.h b/arch/sparc/include/asm/rwsem.h
deleted file mode 100644 (file)
index 069bf4d..0000000
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * rwsem.h: R/W semaphores implemented using CAS
- *
- * Written by David S. Miller (davem@redhat.com), 2001.
- * Derived from asm-i386/rwsem.h
- */
-#ifndef _SPARC64_RWSEM_H
-#define _SPARC64_RWSEM_H
-
-#ifndef _LINUX_RWSEM_H
-#error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead"
-#endif
-
-#ifdef __KERNEL__
-
-#define RWSEM_UNLOCKED_VALUE           0x00000000L
-#define RWSEM_ACTIVE_BIAS              0x00000001L
-#define RWSEM_ACTIVE_MASK              0xffffffffL
-#define RWSEM_WAITING_BIAS             (-RWSEM_ACTIVE_MASK-1)
-#define RWSEM_ACTIVE_READ_BIAS         RWSEM_ACTIVE_BIAS
-#define RWSEM_ACTIVE_WRITE_BIAS                (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
-
-/*
- * lock for reading
- */
-static inline void __down_read(struct rw_semaphore *sem)
-{
-       if (unlikely(atomic64_inc_return((atomic64_t *)(&sem->count)) <= 0L))
-               rwsem_down_read_failed(sem);
-}
-
-static inline int __down_read_trylock(struct rw_semaphore *sem)
-{
-       long tmp;
-
-       while ((tmp = sem->count) >= 0L) {
-               if (tmp == cmpxchg(&sem->count, tmp,
-                                  tmp + RWSEM_ACTIVE_READ_BIAS)) {
-                       return 1;
-               }
-       }
-       return 0;
-}
-
-/*
- * lock for writing
- */
-static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
-{
-       long tmp;
-
-       tmp = atomic64_add_return(RWSEM_ACTIVE_WRITE_BIAS,
-                                 (atomic64_t *)(&sem->count));
-       if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
-               rwsem_down_write_failed(sem);
-}
-
-static inline void __down_write(struct rw_semaphore *sem)
-{
-       __down_write_nested(sem, 0);
-}
-
-static inline int __down_write_trylock(struct rw_semaphore *sem)
-{
-       long tmp;
-
-       tmp = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
-                     RWSEM_ACTIVE_WRITE_BIAS);
-       return tmp == RWSEM_UNLOCKED_VALUE;
-}
-
-/*
- * unlock after reading
- */
-static inline void __up_read(struct rw_semaphore *sem)
-{
-       long tmp;
-
-       tmp = atomic64_dec_return((atomic64_t *)(&sem->count));
-       if (unlikely(tmp < -1L && (tmp & RWSEM_ACTIVE_MASK) == 0L))
-               rwsem_wake(sem);
-}
-
-/*
- * unlock after writing
- */
-static inline void __up_write(struct rw_semaphore *sem)
-{
-       if (unlikely(atomic64_sub_return(RWSEM_ACTIVE_WRITE_BIAS,
-                                        (atomic64_t *)(&sem->count)) < 0L))
-               rwsem_wake(sem);
-}
-
-/*
- * implement atomic add functionality
- */
-static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem)
-{
-       atomic64_add(delta, (atomic64_t *)(&sem->count));
-}
-
-/*
- * downgrade write lock to read lock
- */
-static inline void __downgrade_write(struct rw_semaphore *sem)
-{
-       long tmp;
-
-       tmp = atomic64_add_return(-RWSEM_WAITING_BIAS, (atomic64_t *)(&sem->count));
-       if (tmp < 0L)
-               rwsem_downgrade_wake(sem);
-}
-
-/*
- * implement exchange and add functionality
- */
-static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem)
-{
-       return atomic64_add_return(delta, (atomic64_t *)(&sem->count));
-}
-
-#endif /* __KERNEL__ */
-
-#endif /* _SPARC64_RWSEM_H */
index ceec86eb68e963d9b6d51ca015d8753ee8bd5144..453744c1d34752c20988cf513ff1eef75c3fa657 100644 (file)
@@ -99,26 +99,36 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
 /*
  * lock for writing
  */
-static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
+#define ____down_write(sem, slow_path)                 \
+({                                                     \
+       long tmp;                                       \
+       struct rw_semaphore* ret;                       \
+       asm volatile("# beginning down_write\n\t"       \
+                    LOCK_PREFIX "  xadd      %1,(%3)\n\t"      \
+                    /* adds 0xffff0001, returns the old value */ \
+                    "  test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t" \
+                    /* was the active mask 0 before? */\
+                    "  jz        1f\n"                 \
+                    "  call " slow_path "\n"           \
+                    "1:\n"                             \
+                    "# ending down_write"              \
+                    : "+m" (sem->count), "=d" (tmp), "=a" (ret)        \
+                    : "a" (sem), "1" (RWSEM_ACTIVE_WRITE_BIAS) \
+                    : "memory", "cc");                 \
+       ret;                                            \
+})
+
+static inline void __down_write(struct rw_semaphore *sem)
 {
-       long tmp;
-       asm volatile("# beginning down_write\n\t"
-                    LOCK_PREFIX "  xadd      %1,(%2)\n\t"
-                    /* adds 0xffff0001, returns the old value */
-                    "  test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t"
-                    /* was the active mask 0 before? */
-                    "  jz        1f\n"
-                    "  call call_rwsem_down_write_failed\n"
-                    "1:\n"
-                    "# ending down_write"
-                    : "+m" (sem->count), "=d" (tmp)
-                    : "a" (sem), "1" (RWSEM_ACTIVE_WRITE_BIAS)
-                    : "memory", "cc");
+       ____down_write(sem, "call_rwsem_down_write_failed");
 }
 
-static inline void __down_write(struct rw_semaphore *sem)
+static inline int __down_write_killable(struct rw_semaphore *sem)
 {
-       __down_write_nested(sem, 0);
+       if (IS_ERR(____down_write(sem, "call_rwsem_down_write_failed_killable")))
+               return -EINTR;
+
+       return 0;
 }
 
 /*
index be110efa0096687c1ea35ada3fc439a9cb3f6180..bf2c6074efd2fb97886d974dea2668f8da59c49c 100644 (file)
  * there is contention on the semaphore.
  *
  * %eax contains the semaphore pointer on entry. Save the C-clobbered
- * registers (%eax, %edx and %ecx) except %eax whish is either a return
- * value or just clobbered..
+ * registers (%eax, %edx and %ecx) except %eax which is either a return
+ * value or just gets clobbered. Same is true for %edx so make sure GCC
+ * reloads it after the slow path, by making it hold a temporary, for
+ * example see ____down_write().
  */
 
 #define save_common_regs \
@@ -106,6 +108,16 @@ ENTRY(call_rwsem_down_write_failed)
        ret
 ENDPROC(call_rwsem_down_write_failed)
 
+ENTRY(call_rwsem_down_write_failed_killable)
+       FRAME_BEGIN
+       save_common_regs
+       movq %rax,%rdi
+       call rwsem_down_write_failed_killable
+       restore_common_regs
+       FRAME_END
+       ret
+ENDPROC(call_rwsem_down_write_failed_killable)
+
 ENTRY(call_rwsem_wake)
        FRAME_BEGIN
        /* do nothing if still outstanding active readers */
index b56855a1382a374f8c52632b9aa243a1112a1a06..28cf4c5d65efade019dbefaf809e71d13624e37c 100644 (file)
@@ -22,6 +22,7 @@ generic-y += mm-arch-hooks.h
 generic-y += percpu.h
 generic-y += preempt.h
 generic-y += resource.h
+generic-y += rwsem.h
 generic-y += sections.h
 generic-y += siginfo.h
 generic-y += statfs.h
diff --git a/arch/xtensa/include/asm/rwsem.h b/arch/xtensa/include/asm/rwsem.h
deleted file mode 100644 (file)
index 249619e..0000000
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * include/asm-xtensa/rwsem.h
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Largely copied from include/asm-ppc/rwsem.h
- *
- * Copyright (C) 2001 - 2005 Tensilica Inc.
- */
-
-#ifndef _XTENSA_RWSEM_H
-#define _XTENSA_RWSEM_H
-
-#ifndef _LINUX_RWSEM_H
-#error "Please don't include <asm/rwsem.h> directly, use <linux/rwsem.h> instead."
-#endif
-
-#define RWSEM_UNLOCKED_VALUE           0x00000000
-#define RWSEM_ACTIVE_BIAS              0x00000001
-#define RWSEM_ACTIVE_MASK              0x0000ffff
-#define RWSEM_WAITING_BIAS             (-0x00010000)
-#define RWSEM_ACTIVE_READ_BIAS         RWSEM_ACTIVE_BIAS
-#define RWSEM_ACTIVE_WRITE_BIAS                (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
-
-/*
- * lock for reading
- */
-static inline void __down_read(struct rw_semaphore *sem)
-{
-       if (atomic_add_return(1,(atomic_t *)(&sem->count)) > 0)
-               smp_wmb();
-       else
-               rwsem_down_read_failed(sem);
-}
-
-static inline int __down_read_trylock(struct rw_semaphore *sem)
-{
-       int tmp;
-
-       while ((tmp = sem->count) >= 0) {
-               if (tmp == cmpxchg(&sem->count, tmp,
-                                  tmp + RWSEM_ACTIVE_READ_BIAS)) {
-                       smp_wmb();
-                       return 1;
-               }
-       }
-       return 0;
-}
-
-/*
- * lock for writing
- */
-static inline void __down_write(struct rw_semaphore *sem)
-{
-       int tmp;
-
-       tmp = atomic_add_return(RWSEM_ACTIVE_WRITE_BIAS,
-                               (atomic_t *)(&sem->count));
-       if (tmp == RWSEM_ACTIVE_WRITE_BIAS)
-               smp_wmb();
-       else
-               rwsem_down_write_failed(sem);
-}
-
-static inline int __down_write_trylock(struct rw_semaphore *sem)
-{
-       int tmp;
-
-       tmp = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
-                     RWSEM_ACTIVE_WRITE_BIAS);
-       smp_wmb();
-       return tmp == RWSEM_UNLOCKED_VALUE;
-}
-
-/*
- * unlock after reading
- */
-static inline void __up_read(struct rw_semaphore *sem)
-{
-       int tmp;
-
-       smp_wmb();
-       tmp = atomic_sub_return(1,(atomic_t *)(&sem->count));
-       if (tmp < -1 && (tmp & RWSEM_ACTIVE_MASK) == 0)
-               rwsem_wake(sem);
-}
-
-/*
- * unlock after writing
- */
-static inline void __up_write(struct rw_semaphore *sem)
-{
-       smp_wmb();
-       if (atomic_sub_return(RWSEM_ACTIVE_WRITE_BIAS,
-                             (atomic_t *)(&sem->count)) < 0)
-               rwsem_wake(sem);
-}
-
-/*
- * implement atomic add functionality
- */
-static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
-{
-       atomic_add(delta, (atomic_t *)(&sem->count));
-}
-
-/*
- * downgrade write lock to read lock
- */
-static inline void __downgrade_write(struct rw_semaphore *sem)
-{
-       int tmp;
-
-       smp_wmb();
-       tmp = atomic_add_return(-RWSEM_WAITING_BIAS, (atomic_t *)(&sem->count));
-       if (tmp < 0)
-               rwsem_downgrade_wake(sem);
-}
-
-/*
- * implement exchange and add functionality
- */
-static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)
-{
-       smp_mb();
-       return atomic_add_return(delta, (atomic_t *)(&sem->count));
-}
-
-#endif /* _XTENSA_RWSEM_H */
index d6d5dc98d7da5509347e56737f3944a37c698490..3fc94a046bf584374b15070f68074c64a3daaf14 100644 (file)
@@ -53,7 +53,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
 /*
  * lock for writing
  */
-static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
+static inline void __down_write(struct rw_semaphore *sem)
 {
        long tmp;
 
@@ -63,9 +63,16 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
                rwsem_down_write_failed(sem);
 }
 
-static inline void __down_write(struct rw_semaphore *sem)
+static inline int __down_write_killable(struct rw_semaphore *sem)
 {
-       __down_write_nested(sem, 0);
+       long tmp;
+
+       tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS,
+                                    (atomic_long_t *)&sem->count);
+       if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
+               if (IS_ERR(rwsem_down_write_failed_killable(sem)))
+                       return -EINTR;
+       return 0;
 }
 
 static inline int __down_write_trylock(struct rw_semaphore *sem)
index d10ef06971b57d8783934333612cb196421716ab..f75222ea7f16598920cef736e2c9fcc21d5fd1b7 100644 (file)
@@ -446,6 +446,18 @@ do {                                                               \
        lock_acquired(&(_lock)->dep_map, _RET_IP_);                     \
 } while (0)
 
+#define LOCK_CONTENDED_RETURN(_lock, try, lock)                        \
+({                                                             \
+       int ____err = 0;                                        \
+       if (!try(_lock)) {                                      \
+               lock_contended(&(_lock)->dep_map, _RET_IP_);    \
+               ____err = lock(_lock);                          \
+       }                                                       \
+       if (!____err)                                           \
+               lock_acquired(&(_lock)->dep_map, _RET_IP_);     \
+       ____err;                                                \
+})
+
 #else /* CONFIG_LOCK_STAT */
 
 #define lock_contended(lockdep_map, ip) do {} while (0)
@@ -454,6 +466,9 @@ do {                                                                \
 #define LOCK_CONTENDED(_lock, try, lock) \
        lock(_lock)
 
+#define LOCK_CONTENDED_RETURN(_lock, try, lock) \
+       lock(_lock)
+
 #endif /* CONFIG_LOCK_STAT */
 
 #ifdef CONFIG_LOCKDEP
index 561e8615528d424ae02b3b3bc8efefa2e9d30d43..ae0528b834cd33acf5c22559778581b4f4f9b37b 100644 (file)
@@ -34,7 +34,7 @@ struct rw_semaphore {
 extern void __down_read(struct rw_semaphore *sem);
 extern int __down_read_trylock(struct rw_semaphore *sem);
 extern void __down_write(struct rw_semaphore *sem);
-extern void __down_write_nested(struct rw_semaphore *sem, int subclass);
+extern int __must_check __down_write_killable(struct rw_semaphore *sem);
 extern int __down_write_trylock(struct rw_semaphore *sem);
 extern void __up_read(struct rw_semaphore *sem);
 extern void __up_write(struct rw_semaphore *sem);
index 8f498cdde2802e25667e3143eac3a6945554563c..d1c12d160ace1491ad097477a03aa6820d304046 100644 (file)
@@ -14,6 +14,7 @@
 #include <linux/list.h>
 #include <linux/spinlock.h>
 #include <linux/atomic.h>
+#include <linux/err.h>
 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
 #include <linux/osq_lock.h>
 #endif
@@ -43,6 +44,7 @@ struct rw_semaphore {
 
 extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);
 extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);
+extern struct rw_semaphore *rwsem_down_write_failed_killable(struct rw_semaphore *sem);
 extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *);
 extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem);
 
@@ -116,6 +118,7 @@ extern int down_read_trylock(struct rw_semaphore *sem);
  * lock for writing
  */
 extern void down_write(struct rw_semaphore *sem);
+extern int __must_check down_write_killable(struct rw_semaphore *sem);
 
 /*
  * trylock for writing -- returns 1 if successful, 0 if contention
index 3a504857206536f68fda513fecf63841d2979a06..1591f6b3539fd5120dc6884e5ab392572398ecc1 100644 (file)
@@ -191,11 +191,12 @@ int __down_read_trylock(struct rw_semaphore *sem)
 /*
  * get a write lock on the semaphore
  */
-void __sched __down_write_nested(struct rw_semaphore *sem, int subclass)
+int __sched __down_write_common(struct rw_semaphore *sem, int state)
 {
        struct rwsem_waiter waiter;
        struct task_struct *tsk;
        unsigned long flags;
+       int ret = 0;
 
        raw_spin_lock_irqsave(&sem->wait_lock, flags);
 
@@ -215,21 +216,33 @@ void __sched __down_write_nested(struct rw_semaphore *sem, int subclass)
                 */
                if (sem->count == 0)
                        break;
-               set_task_state(tsk, TASK_UNINTERRUPTIBLE);
+               if (signal_pending_state(state, current)) {
+                       ret = -EINTR;
+                       goto out;
+               }
+               set_task_state(tsk, state);
                raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
                schedule();
                raw_spin_lock_irqsave(&sem->wait_lock, flags);
        }
        /* got the lock */
        sem->count = -1;
+out:
        list_del(&waiter.list);
 
        raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
+
+       return ret;
 }
 
 void __sched __down_write(struct rw_semaphore *sem)
 {
-       __down_write_nested(sem, 0);
+       __down_write_common(sem, TASK_UNINTERRUPTIBLE);
+}
+
+int __sched __down_write_killable(struct rw_semaphore *sem)
+{
+       return __down_write_common(sem, TASK_KILLABLE);
 }
 
 /*
index a4d4de05b2d16707becf6179f8ae2ebed334bead..09e30c6225e5373c523a034f42924b9973fcb7a4 100644 (file)
@@ -433,12 +433,13 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)
 /*
  * Wait until we successfully acquire the write lock
  */
-__visible
-struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem)
+static inline struct rw_semaphore *
+__rwsem_down_write_failed_common(struct rw_semaphore *sem, int state)
 {
        long count;
        bool waiting = true; /* any queued threads before us */
        struct rwsem_waiter waiter;
+       struct rw_semaphore *ret = sem;
 
        /* undo write bias from down_write operation, stop active locking */
        count = rwsem_atomic_update(-RWSEM_ACTIVE_WRITE_BIAS, sem);
@@ -478,7 +479,7 @@ struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem)
                count = rwsem_atomic_update(RWSEM_WAITING_BIAS, sem);
 
        /* wait until we successfully acquire the lock */
-       set_current_state(TASK_UNINTERRUPTIBLE);
+       set_current_state(state);
        while (true) {
                if (rwsem_try_write_lock(count, sem))
                        break;
@@ -486,21 +487,48 @@ struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem)
 
                /* Block until there are no active lockers. */
                do {
+                       if (signal_pending_state(state, current))
+                               goto out_nolock;
+
                        schedule();
-                       set_current_state(TASK_UNINTERRUPTIBLE);
+                       set_current_state(state);
                } while ((count = sem->count) & RWSEM_ACTIVE_MASK);
 
                raw_spin_lock_irq(&sem->wait_lock);
        }
        __set_current_state(TASK_RUNNING);
+       list_del(&waiter.list);
+       raw_spin_unlock_irq(&sem->wait_lock);
 
+       return ret;
+
+out_nolock:
+       __set_current_state(TASK_RUNNING);
+       raw_spin_lock_irq(&sem->wait_lock);
        list_del(&waiter.list);
+       if (list_empty(&sem->wait_list))
+               rwsem_atomic_update(-RWSEM_WAITING_BIAS, sem);
+       else
+               __rwsem_do_wake(sem, RWSEM_WAKE_ANY);
        raw_spin_unlock_irq(&sem->wait_lock);
 
-       return sem;
+       return ERR_PTR(-EINTR);
+}
+
+__visible struct rw_semaphore * __sched
+rwsem_down_write_failed(struct rw_semaphore *sem)
+{
+       return __rwsem_down_write_failed_common(sem, TASK_UNINTERRUPTIBLE);
 }
 EXPORT_SYMBOL(rwsem_down_write_failed);
 
+__visible struct rw_semaphore * __sched
+rwsem_down_write_failed_killable(struct rw_semaphore *sem)
+{
+       return __rwsem_down_write_failed_common(sem, TASK_KILLABLE);
+}
+EXPORT_SYMBOL(rwsem_down_write_failed_killable);
+
 /*
  * handle waking up a waiter on the semaphore
  * - up_read/up_write has decremented the active part of count if we come here
index 205be0ce34de73e8590f2cd4cdb369526b236540..c817216c161521b92cc64ef90e1da1c54e936839 100644 (file)
@@ -54,6 +54,25 @@ void __sched down_write(struct rw_semaphore *sem)
 
 EXPORT_SYMBOL(down_write);
 
+/*
+ * lock for writing
+ */
+int __sched down_write_killable(struct rw_semaphore *sem)
+{
+       might_sleep();
+       rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
+
+       if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) {
+               rwsem_release(&sem->dep_map, 1, _RET_IP_);
+               return -EINTR;
+       }
+
+       rwsem_set_owner(sem);
+       return 0;
+}
+
+EXPORT_SYMBOL(down_write_killable);
+
 /*
  * trylock for writing -- returns 1 if successful, 0 if contention
  */