Commit | Line | Data |
---|---|---|
70af2f8a | 1 | /* |
c7114b4e | 2 | * Queued read/write locks |
70af2f8a WL |
3 | * |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License as published by | |
6 | * the Free Software Foundation; either version 2 of the License, or | |
7 | * (at your option) any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P. | |
15 | * | |
16 | * Authors: Waiman Long <waiman.long@hp.com> | |
17 | */ | |
18 | #include <linux/smp.h> | |
19 | #include <linux/bug.h> | |
20 | #include <linux/cpumask.h> | |
21 | #include <linux/percpu.h> | |
22 | #include <linux/hardirq.h> | |
9ab6055f | 23 | #include <linux/spinlock.h> |
70af2f8a WL |
24 | #include <asm/qrwlock.h> |
25 | ||
70af2f8a | 26 | /** |
f7d71f20 | 27 | * queued_read_lock_slowpath - acquire read lock of a queue rwlock |
70af2f8a WL |
28 | * @lock: Pointer to queue rwlock structure |
29 | */ | |
b519b56e | 30 | void queued_read_lock_slowpath(struct qrwlock *lock) |
70af2f8a | 31 | { |
70af2f8a WL |
32 | /* |
33 | * Readers come here when they cannot get the lock without waiting | |
34 | */ | |
35 | if (unlikely(in_interrupt())) { | |
36 | /* | |
0e06e5be | 37 | * Readers in interrupt context will get the lock immediately |
b519b56e WD |
38 | * if the writer is just waiting (not holding the lock yet), |
39 | * so spin with ACQUIRE semantics until the lock is available | |
40 | * without waiting in the queue. | |
70af2f8a | 41 | */ |
d1331661 | 42 | atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED)); |
70af2f8a WL |
43 | return; |
44 | } | |
45 | atomic_sub(_QR_BIAS, &lock->cnts); | |
46 | ||
47 | /* | |
48 | * Put the reader into the wait queue | |
49 | */ | |
6e1e5196 | 50 | arch_spin_lock(&lock->wait_lock); |
b519b56e | 51 | atomic_add(_QR_BIAS, &lock->cnts); |
70af2f8a WL |
52 | |
53 | /* | |
77e430e3 WD |
54 | * The ACQUIRE semantics of the following spinning code ensure |
55 | * that accesses can't leak upwards out of our subsequent critical | |
56 | * section in the case that the lock is currently held for write. | |
70af2f8a | 57 | */ |
d1331661 | 58 | atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED)); |
70af2f8a WL |
59 | |
60 | /* | |
61 | * Signal the next one in queue to become queue head | |
62 | */ | |
6e1e5196 | 63 | arch_spin_unlock(&lock->wait_lock); |
70af2f8a | 64 | } |
f7d71f20 | 65 | EXPORT_SYMBOL(queued_read_lock_slowpath); |
70af2f8a WL |
66 | |
67 | /** | |
f7d71f20 | 68 | * queued_write_lock_slowpath - acquire write lock of a queue rwlock |
70af2f8a WL |
69 | * @lock : Pointer to queue rwlock structure |
70 | */ | |
f7d71f20 | 71 | void queued_write_lock_slowpath(struct qrwlock *lock) |
70af2f8a | 72 | { |
70af2f8a | 73 | /* Put the writer into the wait queue */ |
6e1e5196 | 74 | arch_spin_lock(&lock->wait_lock); |
70af2f8a WL |
75 | |
76 | /* Try to acquire the lock directly if no reader is present */ | |
77 | if (!atomic_read(&lock->cnts) && | |
77e430e3 | 78 | (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0)) |
70af2f8a WL |
79 | goto unlock; |
80 | ||
d1331661 WD |
81 | /* Set the waiting flag to notify readers that a writer is pending */ |
82 | atomic_add(_QW_WAITING, &lock->cnts); | |
70af2f8a | 83 | |
d1331661 | 84 | /* When no more readers or writers, set the locked flag */ |
b519b56e WD |
85 | do { |
86 | atomic_cond_read_acquire(&lock->cnts, VAL == _QW_WAITING); | |
87 | } while (atomic_cmpxchg_relaxed(&lock->cnts, _QW_WAITING, | |
88 | _QW_LOCKED) != _QW_WAITING); | |
70af2f8a | 89 | unlock: |
6e1e5196 | 90 | arch_spin_unlock(&lock->wait_lock); |
70af2f8a | 91 | } |
f7d71f20 | 92 | EXPORT_SYMBOL(queued_write_lock_slowpath); |