1 // SPDX-License-Identifier: GPL-2.0-only
3 * PREEMPT_RT substitution for spin/rw_locks
5 * spinlocks and rwlocks on RT are based on rtmutexes, with a few twists to
6 * resemble the non RT semantics:
8 * - Contrary to plain rtmutexes, spinlocks and rwlocks are state
9 * preserving. The task state is saved before blocking on the underlying
10 * rtmutex, and restored when the lock has been acquired. Regular wakeups
11 * during that time are redirected to the saved state so no wake up is
14 * - Non RT spin/rwlocks disable preemption and eventually interrupts.
15 * Disabling preemption has the side effect of disabling migration and
16 * preventing RCU grace periods.
18 * The RT substitutions explicitly disable migration and take
19 * rcu_read_lock() across the lock held section.
21 #include <linux/spinlock.h>
22 #include <linux/export.h>
24 #define RT_MUTEX_BUILD_SPINLOCKS
28 * __might_resched() skips the state check as rtlocks are state
29 * preserving. Take RCU nesting into account as spin/read/write_lock() can
30 * legitimately nest into an RCU read side critical section.
32 #define RTLOCK_RESCHED_OFFSETS \
33 (rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT)
35 #define rtlock_might_resched() \
36 __might_resched(__FILE__, __LINE__, RTLOCK_RESCHED_OFFSETS)
38 static __always_inline void rtlock_lock(struct rt_mutex_base *rtm)
40 lockdep_assert(!current->pi_blocked_on);
42 if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
46 static __always_inline void __rt_spin_lock(spinlock_t *lock)
48 rtlock_might_resched();
49 rtlock_lock(&lock->lock);
54 void __sched rt_spin_lock(spinlock_t *lock)
56 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
59 EXPORT_SYMBOL(rt_spin_lock);
61 #ifdef CONFIG_DEBUG_LOCK_ALLOC
62 void __sched rt_spin_lock_nested(spinlock_t *lock, int subclass)
64 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
67 EXPORT_SYMBOL(rt_spin_lock_nested);
69 void __sched rt_spin_lock_nest_lock(spinlock_t *lock,
70 struct lockdep_map *nest_lock)
72 spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
75 EXPORT_SYMBOL(rt_spin_lock_nest_lock);
78 void __sched rt_spin_unlock(spinlock_t *lock)
80 spin_release(&lock->dep_map, _RET_IP_);
84 if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL)))
85 rt_mutex_slowunlock(&lock->lock);
87 EXPORT_SYMBOL(rt_spin_unlock);
90 * Wait for the lock to get unlocked: instead of polling for an unlock
91 * (like raw spinlocks do), lock and unlock, to force the kernel to
92 * schedule if there's contention:
94 void __sched rt_spin_lock_unlock(spinlock_t *lock)
99 EXPORT_SYMBOL(rt_spin_lock_unlock);
101 static __always_inline int __rt_spin_trylock(spinlock_t *lock)
105 if (unlikely(!rt_mutex_cmpxchg_acquire(&lock->lock, NULL, current)))
106 ret = rt_mutex_slowtrylock(&lock->lock);
109 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
116 int __sched rt_spin_trylock(spinlock_t *lock)
118 return __rt_spin_trylock(lock);
120 EXPORT_SYMBOL(rt_spin_trylock);
122 int __sched rt_spin_trylock_bh(spinlock_t *lock)
127 ret = __rt_spin_trylock(lock);
132 EXPORT_SYMBOL(rt_spin_trylock_bh);
134 #ifdef CONFIG_DEBUG_LOCK_ALLOC
135 void __rt_spin_lock_init(spinlock_t *lock, const char *name,
136 struct lock_class_key *key, bool percpu)
138 u8 type = percpu ? LD_LOCK_PERCPU : LD_LOCK_NORMAL;
140 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
141 lockdep_init_map_type(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG,
144 EXPORT_SYMBOL(__rt_spin_lock_init);
148 * RT-specific reader/writer locks
150 #define rwbase_set_and_save_current_state(state) \
151 current_save_and_set_rtlock_wait_state()
153 #define rwbase_restore_current_state() \
154 current_restore_rtlock_saved_state()
156 static __always_inline int
157 rwbase_rtmutex_lock_state(struct rt_mutex_base *rtm, unsigned int state)
159 if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
160 rtlock_slowlock(rtm);
164 static __always_inline int
165 rwbase_rtmutex_slowlock_locked(struct rt_mutex_base *rtm, unsigned int state)
167 rtlock_slowlock_locked(rtm);
171 static __always_inline void rwbase_rtmutex_unlock(struct rt_mutex_base *rtm)
173 if (likely(rt_mutex_cmpxchg_acquire(rtm, current, NULL)))
176 rt_mutex_slowunlock(rtm);
179 static __always_inline int rwbase_rtmutex_trylock(struct rt_mutex_base *rtm)
181 if (likely(rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
184 return rt_mutex_slowtrylock(rtm);
187 #define rwbase_signal_pending_state(state, current) (0)
189 #define rwbase_pre_schedule()
191 #define rwbase_schedule() \
194 #define rwbase_post_schedule()
196 #include "rwbase_rt.c"
198 * The common functions which get wrapped into the rwlock API.
200 int __sched rt_read_trylock(rwlock_t *rwlock)
204 ret = rwbase_read_trylock(&rwlock->rwbase);
206 rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_);
212 EXPORT_SYMBOL(rt_read_trylock);
214 int __sched rt_write_trylock(rwlock_t *rwlock)
218 ret = rwbase_write_trylock(&rwlock->rwbase);
220 rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_);
226 EXPORT_SYMBOL(rt_write_trylock);
228 void __sched rt_read_lock(rwlock_t *rwlock)
230 rtlock_might_resched();
231 rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_);
232 rwbase_read_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
236 EXPORT_SYMBOL(rt_read_lock);
238 void __sched rt_write_lock(rwlock_t *rwlock)
240 rtlock_might_resched();
241 rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
242 rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
246 EXPORT_SYMBOL(rt_write_lock);
248 #ifdef CONFIG_DEBUG_LOCK_ALLOC
249 void __sched rt_write_lock_nested(rwlock_t *rwlock, int subclass)
251 rtlock_might_resched();
252 rwlock_acquire(&rwlock->dep_map, subclass, 0, _RET_IP_);
253 rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
257 EXPORT_SYMBOL(rt_write_lock_nested);
260 void __sched rt_read_unlock(rwlock_t *rwlock)
262 rwlock_release(&rwlock->dep_map, _RET_IP_);
265 rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
267 EXPORT_SYMBOL(rt_read_unlock);
269 void __sched rt_write_unlock(rwlock_t *rwlock)
271 rwlock_release(&rwlock->dep_map, _RET_IP_);
274 rwbase_write_unlock(&rwlock->rwbase);
276 EXPORT_SYMBOL(rt_write_unlock);
278 #ifdef CONFIG_DEBUG_LOCK_ALLOC
279 void __rt_rwlock_init(rwlock_t *rwlock, const char *name,
280 struct lock_class_key *key)
282 debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock));
283 lockdep_init_map_wait(&rwlock->dep_map, name, key, 0, LD_WAIT_CONFIG);
285 EXPORT_SYMBOL(__rt_rwlock_init);