include/linux/spinlock_api_smp.h

   1 #ifndef __LINUX_SPINLOCK_API_SMP_H
   2 #define __LINUX_SPINLOCK_API_SMP_H
   3
   4 #ifndef __LINUX_SPINLOCK_H
   5 # error "please don't include this file directly"
   6 #endif
   7
   8 /*
   9  * include/linux/spinlock_api_smp.h
  10  *
  11  * spinlock API declarations on SMP (and debug)
  12  * (implemented in kernel/spinlock.c)
  13  *
  14  * portions Copyright 2005, Red Hat, Inc., Ingo Molnar
  15  * Released under the General Public License (GPL).
  16  */
  17
  18 int in_lock_functions(unsigned long addr);
  19
  20 #define assert_spin_locked(x)   BUG_ON(!spin_is_locked(x))
  21
  22 void __lockfunc _spin_lock(spinlock_t *lock)            __acquires(lock);
  23 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
  24                                                         __acquires(lock);
  25 void __lockfunc _spin_lock_nest_lock(spinlock_t *lock, struct lockdep_map *map)
  26                                                         __acquires(lock);
  27 void __lockfunc _read_lock(rwlock_t *lock)              __acquires(lock);
  28 void __lockfunc _write_lock(rwlock_t *lock)             __acquires(lock);
  29 void __lockfunc _spin_lock_bh(spinlock_t *lock)         __acquires(lock);
  30 void __lockfunc _read_lock_bh(rwlock_t *lock)           __acquires(lock);
  31 void __lockfunc _write_lock_bh(rwlock_t *lock)          __acquires(lock);
  32 void __lockfunc _spin_lock_irq(spinlock_t *lock)        __acquires(lock);
  33 void __lockfunc _read_lock_irq(rwlock_t *lock)          __acquires(lock);
  34 void __lockfunc _write_lock_irq(rwlock_t *lock)         __acquires(lock);
  35 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
  36                                                         __acquires(lock);
  37 unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
  38                                                         __acquires(lock);
  39 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
  40                                                         __acquires(lock);
  41 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
  42                                                         __acquires(lock);
  43 int __lockfunc _spin_trylock(spinlock_t *lock);
  44 int __lockfunc _read_trylock(rwlock_t *lock);
  45 int __lockfunc _write_trylock(rwlock_t *lock);
  46 int __lockfunc _spin_trylock_bh(spinlock_t *lock);
  47 void __lockfunc _spin_unlock(spinlock_t *lock)          __releases(lock);
  48 void __lockfunc _read_unlock(rwlock_t *lock)            __releases(lock);
  49 void __lockfunc _write_unlock(rwlock_t *lock)           __releases(lock);
  50 void __lockfunc _spin_unlock_bh(spinlock_t *lock)       __releases(lock);
  51 void __lockfunc _read_unlock_bh(rwlock_t *lock)         __releases(lock);
  52 void __lockfunc _write_unlock_bh(rwlock_t *lock)        __releases(lock);
  53 void __lockfunc _spin_unlock_irq(spinlock_t *lock)      __releases(lock);
  54 void __lockfunc _read_unlock_irq(rwlock_t *lock)        __releases(lock);
  55 void __lockfunc _write_unlock_irq(rwlock_t *lock)       __releases(lock);
  56 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
  57                                                         __releases(lock);
  58 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
  59                                                         __releases(lock);
  60 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
  61                                                         __releases(lock);
  62
  63 #ifndef CONFIG_DEBUG_SPINLOCK
  64 #ifndef CONFIG_GENERIC_LOCKBREAK
  65
  66 #ifdef __always_inline__spin_lock
  67 #define _spin_lock(lock) __spin_lock(lock)
  68 #endif
  69
  70 #ifdef __always_inline__read_lock
  71 #define _read_lock(lock) __read_lock(lock)
  72 #endif
  73
  74 #ifdef __always_inline__write_lock
  75 #define _write_lock(lock) __write_lock(lock)
  76 #endif
  77
  78 #ifdef __always_inline__spin_lock_bh
  79 #define _spin_lock_bh(lock) __spin_lock_bh(lock)
  80 #endif
  81
  82 #ifdef __always_inline__read_lock_bh
  83 #define _read_lock_bh(lock) __read_lock_bh(lock)
  84 #endif
  85
  86 #ifdef __always_inline__write_lock_bh
  87 #define _write_lock_bh(lock) __write_lock_bh(lock)
  88 #endif
  89
  90 #ifdef __always_inline__spin_lock_irq
  91 #define _spin_lock_irq(lock) __spin_lock_irq(lock)
  92 #endif
  93
  94 #ifdef __always_inline__read_lock_irq
  95 #define _read_lock_irq(lock) __read_lock_irq(lock)
  96 #endif
  97
  98 #ifdef __always_inline__write_lock_irq
  99 #define _write_lock_irq(lock) __write_lock_irq(lock)
 100 #endif
 101
 102 #ifdef __always_inline__spin_lock_irqsave
 103 #define _spin_lock_irqsave(lock) __spin_lock_irqsave(lock)
 104 #endif
 105
 106 #ifdef __always_inline__read_lock_irqsave
 107 #define _read_lock_irqsave(lock) __read_lock_irqsave(lock)
 108 #endif
 109
 110 #ifdef __always_inline__write_lock_irqsave
 111 #define _write_lock_irqsave(lock) __write_lock_irqsave(lock)
 112 #endif
 113
 114 #endif /* !CONFIG_GENERIC_LOCKBREAK */
 115
 116 #ifdef __always_inline__spin_trylock
 117 #define _spin_trylock(lock) __spin_trylock(lock)
 118 #endif
 119
 120 #ifdef __always_inline__read_trylock
 121 #define _read_trylock(lock) __read_trylock(lock)
 122 #endif
 123
 124 #ifdef __always_inline__write_trylock
 125 #define _write_trylock(lock) __write_trylock(lock)
 126 #endif
 127
 128 #ifdef __always_inline__spin_trylock_bh
 129 #define _spin_trylock_bh(lock) __spin_trylock_bh(lock)
 130 #endif
 131
 132 #ifdef __always_inline__spin_unlock
 133 #define _spin_unlock(lock) __spin_unlock(lock)
 134 #endif
 135
 136 #ifdef __always_inline__read_unlock
 137 #define _read_unlock(lock) __read_unlock(lock)
 138 #endif
 139
 140 #ifdef __always_inline__write_unlock
 141 #define _write_unlock(lock) __write_unlock(lock)
 142 #endif
 143
 144 #ifdef __always_inline__spin_unlock_bh
 145 #define _spin_unlock_bh(lock) __spin_unlock_bh(lock)
 146 #endif
 147
 148 #ifdef __always_inline__read_unlock_bh
 149 #define _read_unlock_bh(lock) __read_unlock_bh(lock)
 150 #endif
 151
 152 #ifdef __always_inline__write_unlock_bh
 153 #define _write_unlock_bh(lock) __write_unlock_bh(lock)
 154 #endif
 155
 156 #ifdef __always_inline__spin_unlock_irq
 157 #define _spin_unlock_irq(lock) __spin_unlock_irq(lock)
 158 #endif
 159
 160 #ifdef __always_inline__read_unlock_irq
 161 #define _read_unlock_irq(lock) __read_unlock_irq(lock)
 162 #endif
 163
 164 #ifdef __always_inline__write_unlock_irq
 165 #define _write_unlock_irq(lock) __write_unlock_irq(lock)
 166 #endif
 167
 168 #ifdef __always_inline__spin_unlock_irqrestore
 169 #define _spin_unlock_irqrestore(lock, flags) __spin_unlock_irqrestore(lock, flags)
 170 #endif
 171
 172 #ifdef __always_inline__read_unlock_irqrestore
 173 #define _read_unlock_irqrestore(lock, flags) __read_unlock_irqrestore(lock, flags)
 174 #endif
 175
 176 #ifdef __always_inline__write_unlock_irqrestore
 177 #define _write_unlock_irqrestore(lock, flags) __write_unlock_irqrestore(lock, flags)
 178 #endif
 179
 180 #endif /* CONFIG_DEBUG_SPINLOCK */
 181
 182 static inline int __spin_trylock(spinlock_t *lock)
 183 {
 184         preempt_disable();
 185         if (_raw_spin_trylock(lock)) {
 186                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
 187                 return 1;
 188         }
 189         preempt_enable();
 190         return 0;
 191 }
 192
 193 static inline int __read_trylock(rwlock_t *lock)
 194 {
 195         preempt_disable();
 196         if (_raw_read_trylock(lock)) {
 197                 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
 198                 return 1;
 199         }
 200         preempt_enable();
 201         return 0;
 202 }
 203
 204 static inline int __write_trylock(rwlock_t *lock)
 205 {
 206         preempt_disable();
 207         if (_raw_write_trylock(lock)) {
 208                 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
 209                 return 1;
 210         }
 211         preempt_enable();
 212         return 0;
 213 }
 214
 215 /*
 216  * If lockdep is enabled then we use the non-preemption spin-ops
 217  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
 218  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
 219  */
 220 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
 221
 222 static inline void __read_lock(rwlock_t *lock)
 223 {
 224         preempt_disable();
 225         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 226         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
 227 }
 228
 229 static inline unsigned long __spin_lock_irqsave(spinlock_t *lock)
 230 {
 231         unsigned long flags;
 232
 233         local_irq_save(flags);
 234         preempt_disable();
 235         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 236         /*
 237          * On lockdep we dont want the hand-coded irq-enable of
 238          * _raw_spin_lock_flags() code, because lockdep assumes
 239          * that interrupts are not re-enabled during lock-acquire:
 240          */
 241 #ifdef CONFIG_LOCKDEP
 242         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 243 #else
 244         _raw_spin_lock_flags(lock, &flags);
 245 #endif
 246         return flags;
 247 }
 248
 249 static inline void __spin_lock_irq(spinlock_t *lock)
 250 {
 251         local_irq_disable();
 252         preempt_disable();
 253         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 254         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 255 }
 256
 257 static inline void __spin_lock_bh(spinlock_t *lock)
 258 {
 259         local_bh_disable();
 260         preempt_disable();
 261         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 262         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 263 }
 264
 265 static inline unsigned long __read_lock_irqsave(rwlock_t *lock)
 266 {
 267         unsigned long flags;
 268
 269         local_irq_save(flags);
 270         preempt_disable();
 271         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 272         LOCK_CONTENDED_FLAGS(lock, _raw_read_trylock, _raw_read_lock,
 273                              _raw_read_lock_flags, &flags);
 274         return flags;
 275 }
 276
 277 static inline void __read_lock_irq(rwlock_t *lock)
 278 {
 279         local_irq_disable();
 280         preempt_disable();
 281         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 282         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
 283 }
 284
 285 static inline void __read_lock_bh(rwlock_t *lock)
 286 {
 287         local_bh_disable();
 288         preempt_disable();
 289         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 290         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
 291 }
 292
 293 static inline unsigned long __write_lock_irqsave(rwlock_t *lock)
 294 {
 295         unsigned long flags;
 296
 297         local_irq_save(flags);
 298         preempt_disable();
 299         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 300         LOCK_CONTENDED_FLAGS(lock, _raw_write_trylock, _raw_write_lock,
 301                              _raw_write_lock_flags, &flags);
 302         return flags;
 303 }
 304
 305 static inline void __write_lock_irq(rwlock_t *lock)
 306 {
 307         local_irq_disable();
 308         preempt_disable();
 309         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 310         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
 311 }
 312
 313 static inline void __write_lock_bh(rwlock_t *lock)
 314 {
 315         local_bh_disable();
 316         preempt_disable();
 317         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 318         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
 319 }
 320
 321 static inline void __spin_lock(spinlock_t *lock)
 322 {
 323         preempt_disable();
 324         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 325         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 326 }
 327
 328 static inline void __write_lock(rwlock_t *lock)
 329 {
 330         preempt_disable();
 331         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 332         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
 333 }
 334
 335 #endif /* CONFIG_PREEMPT */
 336
 337 static inline void __spin_unlock(spinlock_t *lock)
 338 {
 339         spin_release(&lock->dep_map, 1, _RET_IP_);
 340         _raw_spin_unlock(lock);
 341         preempt_enable();
 342 }
 343
 344 static inline void __write_unlock(rwlock_t *lock)
 345 {
 346         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 347         _raw_write_unlock(lock);
 348         preempt_enable();
 349 }
 350
 351 static inline void __read_unlock(rwlock_t *lock)
 352 {
 353         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 354         _raw_read_unlock(lock);
 355         preempt_enable();
 356 }
 357
 358 static inline void __spin_unlock_irqrestore(spinlock_t *lock,
 359                                             unsigned long flags)
 360 {
 361         spin_release(&lock->dep_map, 1, _RET_IP_);
 362         _raw_spin_unlock(lock);
 363         local_irq_restore(flags);
 364         preempt_enable();
 365 }
 366
 367 static inline void __spin_unlock_irq(spinlock_t *lock)
 368 {
 369         spin_release(&lock->dep_map, 1, _RET_IP_);
 370         _raw_spin_unlock(lock);
 371         local_irq_enable();
 372         preempt_enable();
 373 }
 374
 375 static inline void __spin_unlock_bh(spinlock_t *lock)
 376 {
 377         spin_release(&lock->dep_map, 1, _RET_IP_);
 378         _raw_spin_unlock(lock);
 379         preempt_enable_no_resched();
 380         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 381 }
 382
 383 static inline void __read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 384 {
 385         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 386         _raw_read_unlock(lock);
 387         local_irq_restore(flags);
 388         preempt_enable();
 389 }
 390
 391 static inline void __read_unlock_irq(rwlock_t *lock)
 392 {
 393         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 394         _raw_read_unlock(lock);
 395         local_irq_enable();
 396         preempt_enable();
 397 }
 398
 399 static inline void __read_unlock_bh(rwlock_t *lock)
 400 {
 401         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 402         _raw_read_unlock(lock);
 403         preempt_enable_no_resched();
 404         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 405 }
 406
 407 static inline void __write_unlock_irqrestore(rwlock_t *lock,
 408                                              unsigned long flags)
 409 {
 410         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 411         _raw_write_unlock(lock);
 412         local_irq_restore(flags);
 413         preempt_enable();
 414 }
 415
 416 static inline void __write_unlock_irq(rwlock_t *lock)
 417 {
 418         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 419         _raw_write_unlock(lock);
 420         local_irq_enable();
 421         preempt_enable();
 422 }
 423
 424 static inline void __write_unlock_bh(rwlock_t *lock)
 425 {
 426         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 427         _raw_write_unlock(lock);
 428         preempt_enable_no_resched();
 429         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 430 }
 431
 432 static inline int __spin_trylock_bh(spinlock_t *lock)
 433 {
 434         local_bh_disable();
 435         preempt_disable();
 436         if (_raw_spin_trylock(lock)) {
 437                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
 438                 return 1;
 439         }
 440         preempt_enable_no_resched();
 441         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 442         return 0;
 443 }
 444
 445 #endif /* __LINUX_SPINLOCK_API_SMP_H */