Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Copyright (2004) Linus Torvalds | |
3 | * | |
4 | * Author: Zwane Mwaikambo <zwane@fsmlabs.com> | |
5 | * | |
fb1c8f93 IM |
6 | * Copyright (2004, 2005) Ingo Molnar |
7 | * | |
8 | * This file contains the spinlock/rwlock implementations for the | |
9 | * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them) | |
0cb91a22 AK |
10 | * |
11 | * Note that some architectures have special knowledge about the | |
12 | * stack frames of these functions in their profile_pc. If you | |
13 | * change anything significant here that could change the stack | |
14 | * frame contact the architecture maintainers. | |
1da177e4 LT |
15 | */ |
16 | ||
1da177e4 LT |
17 | #include <linux/linkage.h> |
18 | #include <linux/preempt.h> | |
19 | #include <linux/spinlock.h> | |
20 | #include <linux/interrupt.h> | |
8a25d5de | 21 | #include <linux/debug_locks.h> |
1da177e4 LT |
22 | #include <linux/module.h> |
23 | ||
8e13c7b7 TG |
24 | /* |
25 | * If lockdep is enabled then we use the non-preemption spin-ops | |
26 | * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are | |
27 | * not re-enabled during lock-acquire (which the preempt-spin-ops do): | |
28 | */ | |
29 | #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) | |
30 | /* | |
31 | * The __lock_function inlines are taken from | |
32 | * include/linux/spinlock_api_smp.h | |
33 | */ | |
34 | #else | |
35 | /* | |
36 | * We build the __lock_function inlines here. They are too large for | |
37 | * inlining all over the place, but here is only one user per function | |
38 | * which embedds them into the calling _lock_function below. | |
39 | * | |
40 | * This could be a long-held lock. We both prepare to spin for a long | |
41 | * time (making _this_ CPU preemptable if possible), and we also signal | |
42 | * towards that other CPU that it should break the lock ASAP. | |
43 | */ | |
44 | #define BUILD_LOCK_OPS(op, locktype) \ | |
45 | void __lockfunc __##op##_lock(locktype##_t *lock) \ | |
46 | { \ | |
47 | for (;;) { \ | |
48 | preempt_disable(); \ | |
49 | if (likely(_raw_##op##_trylock(lock))) \ | |
50 | break; \ | |
51 | preempt_enable(); \ | |
52 | \ | |
53 | if (!(lock)->break_lock) \ | |
54 | (lock)->break_lock = 1; \ | |
55 | while (!op##_can_lock(lock) && (lock)->break_lock) \ | |
56 | _raw_##op##_relax(&lock->raw_lock); \ | |
57 | } \ | |
58 | (lock)->break_lock = 0; \ | |
59 | } \ | |
60 | \ | |
61 | unsigned long __lockfunc __##op##_lock_irqsave(locktype##_t *lock) \ | |
62 | { \ | |
63 | unsigned long flags; \ | |
64 | \ | |
65 | for (;;) { \ | |
66 | preempt_disable(); \ | |
67 | local_irq_save(flags); \ | |
68 | if (likely(_raw_##op##_trylock(lock))) \ | |
69 | break; \ | |
70 | local_irq_restore(flags); \ | |
71 | preempt_enable(); \ | |
72 | \ | |
73 | if (!(lock)->break_lock) \ | |
74 | (lock)->break_lock = 1; \ | |
75 | while (!op##_can_lock(lock) && (lock)->break_lock) \ | |
76 | _raw_##op##_relax(&lock->raw_lock); \ | |
77 | } \ | |
78 | (lock)->break_lock = 0; \ | |
79 | return flags; \ | |
80 | } \ | |
81 | \ | |
82 | void __lockfunc __##op##_lock_irq(locktype##_t *lock) \ | |
83 | { \ | |
84 | _##op##_lock_irqsave(lock); \ | |
85 | } \ | |
86 | \ | |
87 | void __lockfunc __##op##_lock_bh(locktype##_t *lock) \ | |
88 | { \ | |
89 | unsigned long flags; \ | |
90 | \ | |
91 | /* */ \ | |
92 | /* Careful: we must exclude softirqs too, hence the */ \ | |
93 | /* irq-disabling. We use the generic preemption-aware */ \ | |
94 | /* function: */ \ | |
95 | /**/ \ | |
96 | flags = _##op##_lock_irqsave(lock); \ | |
97 | local_bh_disable(); \ | |
98 | local_irq_restore(flags); \ | |
99 | } \ | |
100 | ||
101 | /* | |
102 | * Build preemption-friendly versions of the following | |
103 | * lock-spinning functions: | |
104 | * | |
105 | * __[spin|read|write]_lock() | |
106 | * __[spin|read|write]_lock_irq() | |
107 | * __[spin|read|write]_lock_irqsave() | |
108 | * __[spin|read|write]_lock_bh() | |
109 | */ | |
110 | BUILD_LOCK_OPS(spin, spinlock); | |
111 | BUILD_LOCK_OPS(read, rwlock); | |
112 | BUILD_LOCK_OPS(write, rwlock); | |
113 | ||
114 | #endif | |
115 | ||
116 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | |
117 | ||
118 | void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass) | |
119 | { | |
120 | preempt_disable(); | |
121 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | |
122 | LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); | |
123 | } | |
124 | EXPORT_SYMBOL(_spin_lock_nested); | |
125 | ||
126 | unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, | |
127 | int subclass) | |
128 | { | |
129 | unsigned long flags; | |
130 | ||
131 | local_irq_save(flags); | |
132 | preempt_disable(); | |
133 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | |
134 | LOCK_CONTENDED_FLAGS(lock, _raw_spin_trylock, _raw_spin_lock, | |
135 | _raw_spin_lock_flags, &flags); | |
136 | return flags; | |
137 | } | |
138 | EXPORT_SYMBOL(_spin_lock_irqsave_nested); | |
139 | ||
140 | void __lockfunc _spin_lock_nest_lock(spinlock_t *lock, | |
141 | struct lockdep_map *nest_lock) | |
142 | { | |
143 | preempt_disable(); | |
144 | spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); | |
145 | LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); | |
146 | } | |
147 | EXPORT_SYMBOL(_spin_lock_nest_lock); | |
148 | ||
149 | #endif | |
150 | ||
6beb0009 | 151 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK |
1da177e4 LT |
152 | int __lockfunc _spin_trylock(spinlock_t *lock) |
153 | { | |
69d0ee73 | 154 | return __spin_trylock(lock); |
1da177e4 LT |
155 | } |
156 | EXPORT_SYMBOL(_spin_trylock); | |
892a7c67 | 157 | #endif |
1da177e4 | 158 | |
6beb0009 | 159 | #ifndef CONFIG_INLINE_READ_TRYLOCK |
1da177e4 LT |
160 | int __lockfunc _read_trylock(rwlock_t *lock) |
161 | { | |
69d0ee73 | 162 | return __read_trylock(lock); |
1da177e4 LT |
163 | } |
164 | EXPORT_SYMBOL(_read_trylock); | |
892a7c67 | 165 | #endif |
1da177e4 | 166 | |
6beb0009 | 167 | #ifndef CONFIG_INLINE_WRITE_TRYLOCK |
1da177e4 LT |
168 | int __lockfunc _write_trylock(rwlock_t *lock) |
169 | { | |
69d0ee73 | 170 | return __write_trylock(lock); |
1da177e4 LT |
171 | } |
172 | EXPORT_SYMBOL(_write_trylock); | |
892a7c67 | 173 | #endif |
1da177e4 | 174 | |
6beb0009 | 175 | #ifndef CONFIG_INLINE_READ_LOCK |
1da177e4 LT |
176 | void __lockfunc _read_lock(rwlock_t *lock) |
177 | { | |
69d0ee73 | 178 | __read_lock(lock); |
1da177e4 LT |
179 | } |
180 | EXPORT_SYMBOL(_read_lock); | |
892a7c67 | 181 | #endif |
1da177e4 | 182 | |
6beb0009 | 183 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE |
1da177e4 LT |
184 | unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock) |
185 | { | |
69d0ee73 | 186 | return __spin_lock_irqsave(lock); |
1da177e4 LT |
187 | } |
188 | EXPORT_SYMBOL(_spin_lock_irqsave); | |
892a7c67 | 189 | #endif |
1da177e4 | 190 | |
6beb0009 | 191 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ |
1da177e4 LT |
192 | void __lockfunc _spin_lock_irq(spinlock_t *lock) |
193 | { | |
69d0ee73 | 194 | __spin_lock_irq(lock); |
1da177e4 LT |
195 | } |
196 | EXPORT_SYMBOL(_spin_lock_irq); | |
892a7c67 | 197 | #endif |
1da177e4 | 198 | |
6beb0009 | 199 | #ifndef CONFIG_INLINE_SPIN_LOCK_BH |
1da177e4 LT |
200 | void __lockfunc _spin_lock_bh(spinlock_t *lock) |
201 | { | |
69d0ee73 | 202 | __spin_lock_bh(lock); |
1da177e4 LT |
203 | } |
204 | EXPORT_SYMBOL(_spin_lock_bh); | |
892a7c67 | 205 | #endif |
1da177e4 | 206 | |
6beb0009 | 207 | #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE |
1da177e4 LT |
208 | unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock) |
209 | { | |
69d0ee73 | 210 | return __read_lock_irqsave(lock); |
1da177e4 LT |
211 | } |
212 | EXPORT_SYMBOL(_read_lock_irqsave); | |
892a7c67 | 213 | #endif |
1da177e4 | 214 | |
6beb0009 | 215 | #ifndef CONFIG_INLINE_READ_LOCK_IRQ |
1da177e4 LT |
216 | void __lockfunc _read_lock_irq(rwlock_t *lock) |
217 | { | |
69d0ee73 | 218 | __read_lock_irq(lock); |
1da177e4 LT |
219 | } |
220 | EXPORT_SYMBOL(_read_lock_irq); | |
892a7c67 | 221 | #endif |
1da177e4 | 222 | |
6beb0009 | 223 | #ifndef CONFIG_INLINE_READ_LOCK_BH |
1da177e4 LT |
224 | void __lockfunc _read_lock_bh(rwlock_t *lock) |
225 | { | |
69d0ee73 | 226 | __read_lock_bh(lock); |
1da177e4 LT |
227 | } |
228 | EXPORT_SYMBOL(_read_lock_bh); | |
892a7c67 | 229 | #endif |
1da177e4 | 230 | |
6beb0009 | 231 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE |
1da177e4 LT |
232 | unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock) |
233 | { | |
69d0ee73 | 234 | return __write_lock_irqsave(lock); |
1da177e4 LT |
235 | } |
236 | EXPORT_SYMBOL(_write_lock_irqsave); | |
892a7c67 | 237 | #endif |
1da177e4 | 238 | |
6beb0009 | 239 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ |
1da177e4 LT |
240 | void __lockfunc _write_lock_irq(rwlock_t *lock) |
241 | { | |
69d0ee73 | 242 | __write_lock_irq(lock); |
1da177e4 LT |
243 | } |
244 | EXPORT_SYMBOL(_write_lock_irq); | |
892a7c67 | 245 | #endif |
1da177e4 | 246 | |
6beb0009 | 247 | #ifndef CONFIG_INLINE_WRITE_LOCK_BH |
1da177e4 LT |
248 | void __lockfunc _write_lock_bh(rwlock_t *lock) |
249 | { | |
69d0ee73 | 250 | __write_lock_bh(lock); |
1da177e4 LT |
251 | } |
252 | EXPORT_SYMBOL(_write_lock_bh); | |
892a7c67 | 253 | #endif |
1da177e4 | 254 | |
6beb0009 | 255 | #ifndef CONFIG_INLINE_SPIN_LOCK |
1da177e4 LT |
256 | void __lockfunc _spin_lock(spinlock_t *lock) |
257 | { | |
69d0ee73 | 258 | __spin_lock(lock); |
1da177e4 | 259 | } |
1da177e4 | 260 | EXPORT_SYMBOL(_spin_lock); |
892a7c67 | 261 | #endif |
1da177e4 | 262 | |
6beb0009 | 263 | #ifndef CONFIG_INLINE_WRITE_LOCK |
1da177e4 LT |
264 | void __lockfunc _write_lock(rwlock_t *lock) |
265 | { | |
69d0ee73 | 266 | __write_lock(lock); |
1da177e4 | 267 | } |
1da177e4 | 268 | EXPORT_SYMBOL(_write_lock); |
892a7c67 | 269 | #endif |
1da177e4 | 270 | |
6beb0009 | 271 | #ifndef CONFIG_INLINE_SPIN_UNLOCK |
1da177e4 LT |
272 | void __lockfunc _spin_unlock(spinlock_t *lock) |
273 | { | |
69d0ee73 | 274 | __spin_unlock(lock); |
1da177e4 LT |
275 | } |
276 | EXPORT_SYMBOL(_spin_unlock); | |
892a7c67 | 277 | #endif |
1da177e4 | 278 | |
6beb0009 | 279 | #ifndef CONFIG_INLINE_WRITE_UNLOCK |
1da177e4 LT |
280 | void __lockfunc _write_unlock(rwlock_t *lock) |
281 | { | |
69d0ee73 | 282 | __write_unlock(lock); |
1da177e4 LT |
283 | } |
284 | EXPORT_SYMBOL(_write_unlock); | |
892a7c67 | 285 | #endif |
1da177e4 | 286 | |
6beb0009 | 287 | #ifndef CONFIG_INLINE_READ_UNLOCK |
1da177e4 LT |
288 | void __lockfunc _read_unlock(rwlock_t *lock) |
289 | { | |
69d0ee73 | 290 | __read_unlock(lock); |
1da177e4 LT |
291 | } |
292 | EXPORT_SYMBOL(_read_unlock); | |
892a7c67 | 293 | #endif |
1da177e4 | 294 | |
6beb0009 | 295 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE |
1da177e4 LT |
296 | void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) |
297 | { | |
69d0ee73 | 298 | __spin_unlock_irqrestore(lock, flags); |
1da177e4 LT |
299 | } |
300 | EXPORT_SYMBOL(_spin_unlock_irqrestore); | |
892a7c67 | 301 | #endif |
1da177e4 | 302 | |
6beb0009 | 303 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ |
1da177e4 LT |
304 | void __lockfunc _spin_unlock_irq(spinlock_t *lock) |
305 | { | |
69d0ee73 | 306 | __spin_unlock_irq(lock); |
1da177e4 LT |
307 | } |
308 | EXPORT_SYMBOL(_spin_unlock_irq); | |
892a7c67 | 309 | #endif |
1da177e4 | 310 | |
6beb0009 | 311 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH |
1da177e4 LT |
312 | void __lockfunc _spin_unlock_bh(spinlock_t *lock) |
313 | { | |
69d0ee73 | 314 | __spin_unlock_bh(lock); |
1da177e4 LT |
315 | } |
316 | EXPORT_SYMBOL(_spin_unlock_bh); | |
892a7c67 | 317 | #endif |
1da177e4 | 318 | |
6beb0009 | 319 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE |
1da177e4 LT |
320 | void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) |
321 | { | |
69d0ee73 | 322 | __read_unlock_irqrestore(lock, flags); |
1da177e4 LT |
323 | } |
324 | EXPORT_SYMBOL(_read_unlock_irqrestore); | |
892a7c67 | 325 | #endif |
1da177e4 | 326 | |
6beb0009 | 327 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ |
1da177e4 LT |
328 | void __lockfunc _read_unlock_irq(rwlock_t *lock) |
329 | { | |
69d0ee73 | 330 | __read_unlock_irq(lock); |
1da177e4 LT |
331 | } |
332 | EXPORT_SYMBOL(_read_unlock_irq); | |
892a7c67 | 333 | #endif |
1da177e4 | 334 | |
6beb0009 | 335 | #ifndef CONFIG_INLINE_READ_UNLOCK_BH |
1da177e4 LT |
336 | void __lockfunc _read_unlock_bh(rwlock_t *lock) |
337 | { | |
69d0ee73 | 338 | __read_unlock_bh(lock); |
1da177e4 LT |
339 | } |
340 | EXPORT_SYMBOL(_read_unlock_bh); | |
892a7c67 | 341 | #endif |
1da177e4 | 342 | |
6beb0009 | 343 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE |
1da177e4 LT |
344 | void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) |
345 | { | |
69d0ee73 | 346 | __write_unlock_irqrestore(lock, flags); |
1da177e4 LT |
347 | } |
348 | EXPORT_SYMBOL(_write_unlock_irqrestore); | |
892a7c67 | 349 | #endif |
1da177e4 | 350 | |
6beb0009 | 351 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ |
1da177e4 LT |
352 | void __lockfunc _write_unlock_irq(rwlock_t *lock) |
353 | { | |
69d0ee73 | 354 | __write_unlock_irq(lock); |
1da177e4 LT |
355 | } |
356 | EXPORT_SYMBOL(_write_unlock_irq); | |
892a7c67 | 357 | #endif |
1da177e4 | 358 | |
6beb0009 | 359 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH |
1da177e4 LT |
360 | void __lockfunc _write_unlock_bh(rwlock_t *lock) |
361 | { | |
69d0ee73 | 362 | __write_unlock_bh(lock); |
1da177e4 LT |
363 | } |
364 | EXPORT_SYMBOL(_write_unlock_bh); | |
892a7c67 | 365 | #endif |
1da177e4 | 366 | |
6beb0009 | 367 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH |
1da177e4 LT |
368 | int __lockfunc _spin_trylock_bh(spinlock_t *lock) |
369 | { | |
69d0ee73 | 370 | return __spin_trylock_bh(lock); |
1da177e4 LT |
371 | } |
372 | EXPORT_SYMBOL(_spin_trylock_bh); | |
892a7c67 | 373 | #endif |
1da177e4 | 374 | |
0764d23c | 375 | notrace int in_lock_functions(unsigned long addr) |
1da177e4 LT |
376 | { |
377 | /* Linker adds these: start and end of __lockfunc functions */ | |
378 | extern char __lock_text_start[], __lock_text_end[]; | |
379 | ||
380 | return addr >= (unsigned long)__lock_text_start | |
381 | && addr < (unsigned long)__lock_text_end; | |
382 | } | |
383 | EXPORT_SYMBOL(in_lock_functions); |