Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
19c5d690 WL |
2 | /* |
3 | * The owner field of the rw_semaphore structure will be set to | |
d7d760ef | 4 | * RWSEM_READER_OWNED when a reader grabs the lock. A writer will clear |
19c5d690 WL |
5 | * the owner field when it unlocks. A reader, on the other hand, will |
6 | * not touch the owner field when it unlocks. | |
7 | * | |
d7d760ef | 8 | * In essence, the owner field now has the following 4 states: |
19c5d690 WL |
9 | * 1) 0 |
10 | * - lock is free or the owner hasn't set the field yet | |
11 | * 2) RWSEM_READER_OWNED | |
12 | * - lock is currently or previously owned by readers (lock is free | |
13 | * or not set by owner yet) | |
d7d760ef WL |
14 | * 3) RWSEM_ANONYMOUSLY_OWNED bit set with some other bits set as well |
15 | * - lock is owned by an anonymous writer, so spinning on the lock | |
16 | * owner should be disabled. | |
17 | * 4) Other non-zero value | |
18 | * - a writer owns the lock and other writers can spin on the lock owner. | |
19c5d690 | 19 | */ |
d7d760ef WL |
20 | #define RWSEM_ANONYMOUSLY_OWNED (1UL << 0) |
21 | #define RWSEM_READER_OWNED ((struct task_struct *)RWSEM_ANONYMOUSLY_OWNED) | |
19c5d690 | 22 | |
5149cbac WL |
23 | #ifdef CONFIG_DEBUG_RWSEMS |
24 | # define DEBUG_RWSEMS_WARN_ON(c) DEBUG_LOCKS_WARN_ON(c) | |
25 | #else | |
26 | # define DEBUG_RWSEMS_WARN_ON(c) | |
27 | #endif | |
28 | ||
7a215f89 | 29 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
fb6a44f3 WL |
30 | /* |
31 | * All writes to owner are protected by WRITE_ONCE() to make sure that | |
32 | * store tearing can't happen as optimistic spinners may read and use | |
33 | * the owner value concurrently without lock. Read from owner, however, | |
34 | * may not need READ_ONCE() as long as the pointer value is only used | |
35 | * for comparison and isn't being dereferenced. | |
36 | */ | |
7a215f89 DB |
37 | static inline void rwsem_set_owner(struct rw_semaphore *sem) |
38 | { | |
fb6a44f3 | 39 | WRITE_ONCE(sem->owner, current); |
7a215f89 DB |
40 | } |
41 | ||
42 | static inline void rwsem_clear_owner(struct rw_semaphore *sem) | |
43 | { | |
fb6a44f3 | 44 | WRITE_ONCE(sem->owner, NULL); |
7a215f89 DB |
45 | } |
46 | ||
19c5d690 WL |
47 | static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) |
48 | { | |
49 | /* | |
50 | * We check the owner value first to make sure that we will only | |
51 | * do a write to the rwsem cacheline when it is really necessary | |
52 | * to minimize cacheline contention. | |
53 | */ | |
5149cbac | 54 | if (READ_ONCE(sem->owner) != RWSEM_READER_OWNED) |
fb6a44f3 | 55 | WRITE_ONCE(sem->owner, RWSEM_READER_OWNED); |
19c5d690 WL |
56 | } |
57 | ||
d7d760ef WL |
58 | /* |
59 | * Return true if the a rwsem waiter can spin on the rwsem's owner | |
60 | * and steal the lock, i.e. the lock is not anonymously owned. | |
61 | * N.B. !owner is considered spinnable. | |
62 | */ | |
63 | static inline bool is_rwsem_owner_spinnable(struct task_struct *owner) | |
19c5d690 | 64 | { |
d7d760ef | 65 | return !((unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED); |
19c5d690 WL |
66 | } |
67 | ||
d7d760ef WL |
68 | /* |
69 | * Return true if rwsem is owned by an anonymous writer or readers. | |
70 | */ | |
71 | static inline bool rwsem_has_anonymous_owner(struct task_struct *owner) | |
19c5d690 | 72 | { |
d7d760ef | 73 | return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED; |
19c5d690 | 74 | } |
7a215f89 DB |
75 | #else |
76 | static inline void rwsem_set_owner(struct rw_semaphore *sem) | |
77 | { | |
78 | } | |
79 | ||
80 | static inline void rwsem_clear_owner(struct rw_semaphore *sem) | |
81 | { | |
82 | } | |
19c5d690 WL |
83 | |
84 | static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) | |
85 | { | |
86 | } | |
7a215f89 | 87 | #endif |