1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __LINUX_SPINLOCK_H
3 #define __LINUX_SPINLOCK_H
4 #define __LINUX_INSIDE_SPINLOCK_H
7 * include/linux/spinlock.h - generic spinlock/rwlock declarations
9 * here's the role of the various spinlock/rwlock related include files:
13 * asm/spinlock_types.h: contains the arch_spinlock_t/arch_rwlock_t and the
16 * linux/spinlock_types_raw:
17 * The raw types and initializers
18 * linux/spinlock_types.h:
19 * defines the generic type and initializers
21 * asm/spinlock.h: contains the arch_spin_*()/etc. lowlevel
22 * implementations, mostly inline assembly code
24 * (also included on UP-debug builds:)
26 * linux/spinlock_api_smp.h:
27 * contains the prototypes for the _spin_*() APIs.
29 * linux/spinlock.h: builds the final spin_*() APIs.
33 * linux/spinlock_type_up.h:
34 * contains the generic, simplified UP spinlock type.
35 * (which is an empty structure on non-debug builds)
37 * linux/spinlock_types_raw:
38 * The raw RT types and initializers
39 * linux/spinlock_types.h:
40 * defines the generic type and initializers
42 * linux/spinlock_up.h:
43 * contains the arch_spin_*()/etc. version of UP
44 * builds. (which are NOPs on non-debug, non-preempt
47 * (included on UP-non-debug builds:)
49 * linux/spinlock_api_up.h:
50 * builds the _spin_*() APIs.
52 * linux/spinlock.h: builds the final spin_*() APIs.
55 #include <linux/typecheck.h>
56 #include <linux/preempt.h>
57 #include <linux/linkage.h>
58 #include <linux/compiler.h>
59 #include <linux/irqflags.h>
60 #include <linux/thread_info.h>
61 #include <linux/stringify.h>
62 #include <linux/bottom_half.h>
63 #include <linux/lockdep.h>
64 #include <asm/barrier.h>
65 #include <asm/mmiowb.h>
69 * Must define these before including other files, inline functions need them
71 #define LOCK_SECTION_NAME ".text..lock."KBUILD_BASENAME
73 #define LOCK_SECTION_START(extra) \
76 ".ifndef " LOCK_SECTION_NAME "\n\t" \
77 LOCK_SECTION_NAME ":\n\t" \
80 #define LOCK_SECTION_END \
83 #define __lockfunc __section(".spinlock.text")
86 * Pull the arch_spinlock_t and arch_rwlock_t definitions:
88 #include <linux/spinlock_types.h>
91 * Pull the arch_spin*() functions/declarations (UP-nondebug doesn't need them):
94 # include <asm/spinlock.h>
96 # include <linux/spinlock_up.h>
99 #ifdef CONFIG_DEBUG_SPINLOCK
100 extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
101 struct lock_class_key *key, short inner);
103 # define raw_spin_lock_init(lock) \
105 static struct lock_class_key __key; \
107 __raw_spin_lock_init((lock), #lock, &__key, LD_WAIT_SPIN); \
111 # define raw_spin_lock_init(lock) \
112 do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0)
115 #define raw_spin_is_locked(lock) arch_spin_is_locked(&(lock)->raw_lock)
117 #ifdef arch_spin_is_contended
118 #define raw_spin_is_contended(lock) arch_spin_is_contended(&(lock)->raw_lock)
120 #define raw_spin_is_contended(lock) (((void)(lock), 0))
121 #endif /*arch_spin_is_contended*/
124 * smp_mb__after_spinlock() provides the equivalent of a full memory barrier
125 * between program-order earlier lock acquisitions and program-order later
128 * This guarantees that the following two properties hold:
130 * 1) Given the snippet:
136 * WRITE_ONCE(X, 1); WRITE_ONCE(Y, 1);
137 * spin_lock(S); smp_mb();
138 * smp_mb__after_spinlock(); r1 = READ_ONCE(X);
142 * it is forbidden that CPU0 does not observe CPU1's store to Y (r0 = 0)
143 * and CPU1 does not observe CPU0's store to X (r1 = 0); see the comments
144 * preceding the call to smp_mb__after_spinlock() in __schedule() and in
147 * 2) Given the snippet:
153 * spin_lock(S); spin_lock(S); r1 = READ_ONCE(Y);
154 * WRITE_ONCE(X, 1); smp_mb__after_spinlock(); smp_rmb();
155 * spin_unlock(S); r0 = READ_ONCE(X); r2 = READ_ONCE(X);
159 * it is forbidden that CPU0's critical section executes before CPU1's
160 * critical section (r0 = 1), CPU2 observes CPU1's store to Y (r1 = 1)
161 * and CPU2 does not observe CPU0's store to X (r2 = 0); see the comments
162 * preceding the calls to smp_rmb() in try_to_wake_up() for similar
163 * snippets but "projected" onto two CPUs.
165 * Property (2) upgrades the lock to an RCsc lock.
167 * Since most load-store architectures implement ACQUIRE with an smp_mb() after
168 * the LL/SC loop, they need no further barriers. Similarly all our TSO
169 * architectures imply an smp_mb() for each atomic instruction and equally don't
172 * Architectures that can implement ACQUIRE better need to take care.
174 #ifndef smp_mb__after_spinlock
175 #define smp_mb__after_spinlock() kcsan_mb()
178 #ifdef CONFIG_DEBUG_SPINLOCK
179 extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock);
180 extern int do_raw_spin_trylock(raw_spinlock_t *lock);
181 extern void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock);
183 static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock)
186 arch_spin_lock(&lock->raw_lock);
190 static inline int do_raw_spin_trylock(raw_spinlock_t *lock)
192 int ret = arch_spin_trylock(&(lock)->raw_lock);
200 static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
202 mmiowb_spin_unlock();
203 arch_spin_unlock(&lock->raw_lock);
209 * Define the various spin_lock methods. Note we define these
210 * regardless of whether CONFIG_SMP or CONFIG_PREEMPTION are set. The
211 * various methods are defined as nops in the case they are not
214 #define raw_spin_trylock(lock) __cond_lock(lock, _raw_spin_trylock(lock))
216 #define raw_spin_lock(lock) _raw_spin_lock(lock)
218 #ifdef CONFIG_DEBUG_LOCK_ALLOC
219 # define raw_spin_lock_nested(lock, subclass) \
220 _raw_spin_lock_nested(lock, subclass)
222 # define raw_spin_lock_nest_lock(lock, nest_lock) \
224 typecheck(struct lockdep_map *, &(nest_lock)->dep_map);\
225 _raw_spin_lock_nest_lock(lock, &(nest_lock)->dep_map); \
229 * Always evaluate the 'subclass' argument to avoid that the compiler
230 * warns about set-but-not-used variables when building with
231 * CONFIG_DEBUG_LOCK_ALLOC=n and with W=1.
233 # define raw_spin_lock_nested(lock, subclass) \
234 _raw_spin_lock(((void)(subclass), (lock)))
235 # define raw_spin_lock_nest_lock(lock, nest_lock) _raw_spin_lock(lock)
238 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
240 #define raw_spin_lock_irqsave(lock, flags) \
242 typecheck(unsigned long, flags); \
243 flags = _raw_spin_lock_irqsave(lock); \
246 #ifdef CONFIG_DEBUG_LOCK_ALLOC
247 #define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
249 typecheck(unsigned long, flags); \
250 flags = _raw_spin_lock_irqsave_nested(lock, subclass); \
253 #define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
255 typecheck(unsigned long, flags); \
256 flags = _raw_spin_lock_irqsave(lock); \
262 #define raw_spin_lock_irqsave(lock, flags) \
264 typecheck(unsigned long, flags); \
265 _raw_spin_lock_irqsave(lock, flags); \
268 #define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
269 raw_spin_lock_irqsave(lock, flags)
273 #define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock)
274 #define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock)
275 #define raw_spin_unlock(lock) _raw_spin_unlock(lock)
276 #define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)
278 #define raw_spin_unlock_irqrestore(lock, flags) \
280 typecheck(unsigned long, flags); \
281 _raw_spin_unlock_irqrestore(lock, flags); \
283 #define raw_spin_unlock_bh(lock) _raw_spin_unlock_bh(lock)
285 #define raw_spin_trylock_bh(lock) \
286 __cond_lock(lock, _raw_spin_trylock_bh(lock))
288 #define raw_spin_trylock_irq(lock) \
290 local_irq_disable(); \
291 raw_spin_trylock(lock) ? \
292 1 : ({ local_irq_enable(); 0; }); \
295 #define raw_spin_trylock_irqsave(lock, flags) \
297 local_irq_save(flags); \
298 raw_spin_trylock(lock) ? \
299 1 : ({ local_irq_restore(flags); 0; }); \
302 #ifndef CONFIG_PREEMPT_RT
303 /* Include rwlock functions for !RT */
304 #include <linux/rwlock.h>
308 * Pull the _spin_*()/_read_*()/_write_*() functions/declarations:
310 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
311 # include <linux/spinlock_api_smp.h>
313 # include <linux/spinlock_api_up.h>
316 /* Non PREEMPT_RT kernel, map to raw spinlocks: */
317 #ifndef CONFIG_PREEMPT_RT
320 * Map the spin_lock functions to the raw variants for PREEMPT_RT=n
323 static __always_inline raw_spinlock_t *spinlock_check(spinlock_t *lock)
328 #ifdef CONFIG_DEBUG_SPINLOCK
330 # define spin_lock_init(lock) \
332 static struct lock_class_key __key; \
334 __raw_spin_lock_init(spinlock_check(lock), \
335 #lock, &__key, LD_WAIT_CONFIG); \
340 # define spin_lock_init(_lock) \
342 spinlock_check(_lock); \
343 *(_lock) = __SPIN_LOCK_UNLOCKED(_lock); \
348 static __always_inline void spin_lock(spinlock_t *lock)
350 raw_spin_lock(&lock->rlock);
353 static __always_inline void spin_lock_bh(spinlock_t *lock)
355 raw_spin_lock_bh(&lock->rlock);
358 static __always_inline int spin_trylock(spinlock_t *lock)
360 return raw_spin_trylock(&lock->rlock);
363 #define spin_lock_nested(lock, subclass) \
365 raw_spin_lock_nested(spinlock_check(lock), subclass); \
368 #define spin_lock_nest_lock(lock, nest_lock) \
370 raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock); \
373 static __always_inline void spin_lock_irq(spinlock_t *lock)
375 raw_spin_lock_irq(&lock->rlock);
378 #define spin_lock_irqsave(lock, flags) \
380 raw_spin_lock_irqsave(spinlock_check(lock), flags); \
383 #define spin_lock_irqsave_nested(lock, flags, subclass) \
385 raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \
388 static __always_inline void spin_unlock(spinlock_t *lock)
390 raw_spin_unlock(&lock->rlock);
393 static __always_inline void spin_unlock_bh(spinlock_t *lock)
395 raw_spin_unlock_bh(&lock->rlock);
398 static __always_inline void spin_unlock_irq(spinlock_t *lock)
400 raw_spin_unlock_irq(&lock->rlock);
403 static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
405 raw_spin_unlock_irqrestore(&lock->rlock, flags);
408 static __always_inline int spin_trylock_bh(spinlock_t *lock)
410 return raw_spin_trylock_bh(&lock->rlock);
413 static __always_inline int spin_trylock_irq(spinlock_t *lock)
415 return raw_spin_trylock_irq(&lock->rlock);
418 #define spin_trylock_irqsave(lock, flags) \
420 raw_spin_trylock_irqsave(spinlock_check(lock), flags); \
424 * spin_is_locked() - Check whether a spinlock is locked.
425 * @lock: Pointer to the spinlock.
427 * This function is NOT required to provide any memory ordering
428 * guarantees; it could be used for debugging purposes or, when
429 * additional synchronization is needed, accompanied with other
430 * constructs (memory barriers) enforcing the synchronization.
432 * Returns: 1 if @lock is locked, 0 otherwise.
434 * Note that the function only tells you that the spinlock is
435 * seen to be locked, not that it is locked on your CPU.
437 * Further, on CONFIG_SMP=n builds with CONFIG_DEBUG_SPINLOCK=n,
438 * the return value is always 0 (see include/linux/spinlock_up.h).
439 * Therefore you should not rely heavily on the return value.
441 static __always_inline int spin_is_locked(spinlock_t *lock)
443 return raw_spin_is_locked(&lock->rlock);
446 static __always_inline int spin_is_contended(spinlock_t *lock)
448 return raw_spin_is_contended(&lock->rlock);
451 #define assert_spin_locked(lock) assert_raw_spin_locked(&(lock)->rlock)
453 #else /* !CONFIG_PREEMPT_RT */
454 # include <linux/spinlock_rt.h>
455 #endif /* CONFIG_PREEMPT_RT */
458 * Pull the atomic_t declaration:
459 * (asm-mips/atomic.h needs above definitions)
461 #include <linux/atomic.h>
463 * atomic_dec_and_lock - lock on reaching reference count zero
464 * @atomic: the atomic counter
465 * @lock: the spinlock in question
467 * Decrements @atomic by 1. If the result is 0, returns true and locks
468 * @lock. Returns false for all other cases.
470 extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
471 #define atomic_dec_and_lock(atomic, lock) \
472 __cond_lock(lock, _atomic_dec_and_lock(atomic, lock))
474 extern int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
475 unsigned long *flags);
476 #define atomic_dec_and_lock_irqsave(atomic, lock, flags) \
477 __cond_lock(lock, _atomic_dec_and_lock_irqsave(atomic, lock, &(flags)))
479 extern int _atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock);
480 #define atomic_dec_and_raw_lock(atomic, lock) \
481 __cond_lock(lock, _atomic_dec_and_raw_lock(atomic, lock))
483 extern int _atomic_dec_and_raw_lock_irqsave(atomic_t *atomic, raw_spinlock_t *lock,
484 unsigned long *flags);
485 #define atomic_dec_and_raw_lock_irqsave(atomic, lock, flags) \
486 __cond_lock(lock, _atomic_dec_and_raw_lock_irqsave(atomic, lock, &(flags)))
488 int __alloc_bucket_spinlocks(spinlock_t **locks, unsigned int *lock_mask,
489 size_t max_size, unsigned int cpu_mult,
490 gfp_t gfp, const char *name,
491 struct lock_class_key *key);
493 #define alloc_bucket_spinlocks(locks, lock_mask, max_size, cpu_mult, gfp) \
495 static struct lock_class_key key; \
498 ret = __alloc_bucket_spinlocks(locks, lock_mask, max_size, \
499 cpu_mult, gfp, #locks, &key); \
503 void free_bucket_spinlocks(spinlock_t *locks);
505 #undef __LINUX_INSIDE_SPINLOCK_H
506 #endif /* __LINUX_SPINLOCK_H */