Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
62ac665f MP |
2 | #ifndef _LINUX_PERCPU_RWSEM_H |
3 | #define _LINUX_PERCPU_RWSEM_H | |
4 | ||
9390ef0c | 5 | #include <linux/atomic.h> |
a1fd3e24 | 6 | #include <linux/rwsem.h> |
62ac665f | 7 | #include <linux/percpu.h> |
52b94129 | 8 | #include <linux/rcuwait.h> |
001dac62 | 9 | #include <linux/rcu_sync.h> |
8ebe3473 | 10 | #include <linux/lockdep.h> |
62ac665f MP |
11 | |
12 | struct percpu_rw_semaphore { | |
001dac62 | 13 | struct rcu_sync rss; |
80127a39 | 14 | unsigned int __percpu *read_count; |
52b94129 DB |
15 | struct rw_semaphore rw_sem; /* slowpath */ |
16 | struct rcuwait writer; /* blocked writer */ | |
80127a39 | 17 | int readers_block; |
1751060e PZ |
18 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
19 | struct lockdep_map dep_map; | |
20 | #endif | |
62ac665f MP |
21 | }; |
22 | ||
1751060e PZ |
23 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
24 | #define __PERCPU_RWSEM_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname }, | |
25 | #else | |
26 | #define __PERCPU_RWSEM_DEP_MAP_INIT(lockname) | |
27 | #endif | |
28 | ||
3f2947b7 | 29 | #define __DEFINE_PERCPU_RWSEM(name, is_static) \ |
11d9684c | 30 | static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name); \ |
3f2947b7 | 31 | is_static struct percpu_rw_semaphore name = { \ |
95bf33b5 | 32 | .rss = __RCU_SYNC_INITIALIZER(name.rss), \ |
11d9684c PZ |
33 | .read_count = &__percpu_rwsem_rc_##name, \ |
34 | .rw_sem = __RWSEM_INITIALIZER(name.rw_sem), \ | |
52b94129 | 35 | .writer = __RCUWAIT_INITIALIZER(name.writer), \ |
1751060e | 36 | __PERCPU_RWSEM_DEP_MAP_INIT(name) \ |
11d9684c | 37 | } |
1751060e | 38 | |
3f2947b7 ON |
39 | #define DEFINE_PERCPU_RWSEM(name) \ |
40 | __DEFINE_PERCPU_RWSEM(name, /* not static */) | |
41 | #define DEFINE_STATIC_PERCPU_RWSEM(name) \ | |
42 | __DEFINE_PERCPU_RWSEM(name, static) | |
11d9684c | 43 | |
206c98ff | 44 | extern bool __percpu_down_read(struct percpu_rw_semaphore *, bool); |
80127a39 PZ |
45 | extern void __percpu_up_read(struct percpu_rw_semaphore *); |
46 | ||
02e525b2 | 47 | static inline void percpu_down_read(struct percpu_rw_semaphore *sem) |
80127a39 PZ |
48 | { |
49 | might_sleep(); | |
50 | ||
1751060e | 51 | rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); |
80127a39 PZ |
52 | |
53 | preempt_disable(); | |
54 | /* | |
55 | * We are in an RCU-sched read-side critical section, so the writer | |
56 | * cannot both change sem->state from readers_fast and start checking | |
57 | * counters while we are here. So if we see !sem->state, we know that | |
58 | * the writer won't be checking until we're past the preempt_enable() | |
e3e74054 | 59 | * and that once the synchronize_rcu() is done, the writer will see |
80127a39 PZ |
60 | * anything we did within this RCU-sched read-size critical section. |
61 | */ | |
71365d40 PZ |
62 | if (likely(rcu_sync_is_idle(&sem->rss))) |
63 | __this_cpu_inc(*sem->read_count); | |
64 | else | |
80127a39 | 65 | __percpu_down_read(sem, false); /* Unconditional memory barrier */ |
80127a39 | 66 | /* |
02e525b2 | 67 | * The preempt_enable() prevents the compiler from |
80127a39 PZ |
68 | * bleeding the critical section out. |
69 | */ | |
259d69b7 PZ |
70 | preempt_enable(); |
71 | } | |
72 | ||
206c98ff | 73 | static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem) |
80127a39 | 74 | { |
206c98ff | 75 | bool ret = true; |
80127a39 PZ |
76 | |
77 | preempt_disable(); | |
78 | /* | |
79 | * Same as in percpu_down_read(). | |
80 | */ | |
71365d40 PZ |
81 | if (likely(rcu_sync_is_idle(&sem->rss))) |
82 | __this_cpu_inc(*sem->read_count); | |
83 | else | |
80127a39 PZ |
84 | ret = __percpu_down_read(sem, true); /* Unconditional memory barrier */ |
85 | preempt_enable(); | |
86 | /* | |
87 | * The barrier() from preempt_enable() prevents the compiler from | |
88 | * bleeding the critical section out. | |
89 | */ | |
90 | ||
91 | if (ret) | |
1751060e | 92 | rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); |
80127a39 PZ |
93 | |
94 | return ret; | |
95 | } | |
96 | ||
02e525b2 | 97 | static inline void percpu_up_read(struct percpu_rw_semaphore *sem) |
80127a39 | 98 | { |
1751060e PZ |
99 | rwsem_release(&sem->dep_map, _RET_IP_); |
100 | ||
02e525b2 | 101 | preempt_disable(); |
80127a39 PZ |
102 | /* |
103 | * Same as in percpu_down_read(). | |
104 | */ | |
105 | if (likely(rcu_sync_is_idle(&sem->rss))) | |
106 | __this_cpu_dec(*sem->read_count); | |
107 | else | |
108 | __percpu_up_read(sem); /* Unconditional memory barrier */ | |
109 | preempt_enable(); | |
80127a39 | 110 | } |
5c1eabe6 | 111 | |
a1fd3e24 ON |
112 | extern void percpu_down_write(struct percpu_rw_semaphore *); |
113 | extern void percpu_up_write(struct percpu_rw_semaphore *); | |
62ac665f | 114 | |
8ebe3473 ON |
115 | extern int __percpu_init_rwsem(struct percpu_rw_semaphore *, |
116 | const char *, struct lock_class_key *); | |
80127a39 | 117 | |
a1fd3e24 | 118 | extern void percpu_free_rwsem(struct percpu_rw_semaphore *); |
62ac665f | 119 | |
80127a39 | 120 | #define percpu_init_rwsem(sem) \ |
8ebe3473 ON |
121 | ({ \ |
122 | static struct lock_class_key rwsem_key; \ | |
80127a39 | 123 | __percpu_init_rwsem(sem, #sem, &rwsem_key); \ |
8ebe3473 ON |
124 | }) |
125 | ||
1751060e PZ |
126 | #define percpu_rwsem_is_held(sem) lockdep_is_held(sem) |
127 | #define percpu_rwsem_assert_held(sem) lockdep_assert_held(sem) | |
11d9684c | 128 | |
55cc1565 ON |
129 | static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem, |
130 | bool read, unsigned long ip) | |
131 | { | |
1751060e | 132 | lock_release(&sem->dep_map, ip); |
55cc1565 ON |
133 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
134 | if (!read) | |
94a9717b | 135 | atomic_long_set(&sem->rw_sem.owner, RWSEM_OWNER_UNKNOWN); |
55cc1565 ON |
136 | #endif |
137 | } | |
138 | ||
139 | static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem, | |
140 | bool read, unsigned long ip) | |
141 | { | |
1751060e | 142 | lock_acquire(&sem->dep_map, 0, 1, read, 1, NULL, ip); |
5a817641 WL |
143 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
144 | if (!read) | |
94a9717b | 145 | atomic_long_set(&sem->rw_sem.owner, (long)current); |
5a817641 | 146 | #endif |
55cc1565 ON |
147 | } |
148 | ||
62ac665f | 149 | #endif |