Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Copyright (2004) Linus Torvalds | |
3 | * | |
4 | * Author: Zwane Mwaikambo <zwane@fsmlabs.com> | |
5 | * | |
fb1c8f93 IM |
6 | * Copyright (2004, 2005) Ingo Molnar |
7 | * | |
8 | * This file contains the spinlock/rwlock implementations for the | |
9 | * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them) | |
0cb91a22 AK |
10 | * |
11 | * Note that some architectures have special knowledge about the | |
12 | * stack frames of these functions in their profile_pc. If you | |
13 | * change anything significant here that could change the stack | |
14 | * frame contact the architecture maintainers. | |
1da177e4 LT |
15 | */ |
16 | ||
1da177e4 LT |
17 | #include <linux/linkage.h> |
18 | #include <linux/preempt.h> | |
19 | #include <linux/spinlock.h> | |
20 | #include <linux/interrupt.h> | |
8a25d5de | 21 | #include <linux/debug_locks.h> |
1da177e4 LT |
22 | #include <linux/module.h> |
23 | ||
8e13c7b7 TG |
24 | /* |
25 | * If lockdep is enabled then we use the non-preemption spin-ops | |
26 | * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are | |
27 | * not re-enabled during lock-acquire (which the preempt-spin-ops do): | |
28 | */ | |
29 | #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) | |
30 | /* | |
31 | * The __lock_function inlines are taken from | |
32 | * include/linux/spinlock_api_smp.h | |
33 | */ | |
34 | #else | |
35 | /* | |
36 | * We build the __lock_function inlines here. They are too large for | |
37 | * inlining all over the place, but here is only one user per function | |
38 | * which embedds them into the calling _lock_function below. | |
39 | * | |
40 | * This could be a long-held lock. We both prepare to spin for a long | |
41 | * time (making _this_ CPU preemptable if possible), and we also signal | |
42 | * towards that other CPU that it should break the lock ASAP. | |
43 | */ | |
44 | #define BUILD_LOCK_OPS(op, locktype) \ | |
45 | void __lockfunc __##op##_lock(locktype##_t *lock) \ | |
46 | { \ | |
47 | for (;;) { \ | |
48 | preempt_disable(); \ | |
49 | if (likely(_raw_##op##_trylock(lock))) \ | |
50 | break; \ | |
51 | preempt_enable(); \ | |
52 | \ | |
53 | if (!(lock)->break_lock) \ | |
54 | (lock)->break_lock = 1; \ | |
55 | while (!op##_can_lock(lock) && (lock)->break_lock) \ | |
0199c4e6 | 56 | arch_##op##_relax(&lock->raw_lock); \ |
8e13c7b7 TG |
57 | } \ |
58 | (lock)->break_lock = 0; \ | |
59 | } \ | |
60 | \ | |
61 | unsigned long __lockfunc __##op##_lock_irqsave(locktype##_t *lock) \ | |
62 | { \ | |
63 | unsigned long flags; \ | |
64 | \ | |
65 | for (;;) { \ | |
66 | preempt_disable(); \ | |
67 | local_irq_save(flags); \ | |
68 | if (likely(_raw_##op##_trylock(lock))) \ | |
69 | break; \ | |
70 | local_irq_restore(flags); \ | |
71 | preempt_enable(); \ | |
72 | \ | |
73 | if (!(lock)->break_lock) \ | |
74 | (lock)->break_lock = 1; \ | |
75 | while (!op##_can_lock(lock) && (lock)->break_lock) \ | |
0199c4e6 | 76 | arch_##op##_relax(&lock->raw_lock); \ |
8e13c7b7 TG |
77 | } \ |
78 | (lock)->break_lock = 0; \ | |
79 | return flags; \ | |
80 | } \ | |
81 | \ | |
82 | void __lockfunc __##op##_lock_irq(locktype##_t *lock) \ | |
83 | { \ | |
84 | _##op##_lock_irqsave(lock); \ | |
85 | } \ | |
86 | \ | |
87 | void __lockfunc __##op##_lock_bh(locktype##_t *lock) \ | |
88 | { \ | |
89 | unsigned long flags; \ | |
90 | \ | |
91 | /* */ \ | |
92 | /* Careful: we must exclude softirqs too, hence the */ \ | |
93 | /* irq-disabling. We use the generic preemption-aware */ \ | |
94 | /* function: */ \ | |
95 | /**/ \ | |
96 | flags = _##op##_lock_irqsave(lock); \ | |
97 | local_bh_disable(); \ | |
98 | local_irq_restore(flags); \ | |
99 | } \ | |
100 | ||
101 | /* | |
102 | * Build preemption-friendly versions of the following | |
103 | * lock-spinning functions: | |
104 | * | |
105 | * __[spin|read|write]_lock() | |
106 | * __[spin|read|write]_lock_irq() | |
107 | * __[spin|read|write]_lock_irqsave() | |
108 | * __[spin|read|write]_lock_bh() | |
109 | */ | |
110 | BUILD_LOCK_OPS(spin, spinlock); | |
111 | BUILD_LOCK_OPS(read, rwlock); | |
112 | BUILD_LOCK_OPS(write, rwlock); | |
113 | ||
114 | #endif | |
115 | ||
6beb0009 | 116 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK |
1da177e4 LT |
117 | int __lockfunc _spin_trylock(spinlock_t *lock) |
118 | { | |
69d0ee73 | 119 | return __spin_trylock(lock); |
1da177e4 LT |
120 | } |
121 | EXPORT_SYMBOL(_spin_trylock); | |
892a7c67 | 122 | #endif |
1da177e4 | 123 | |
b7b40ade TG |
124 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH |
125 | int __lockfunc _spin_trylock_bh(spinlock_t *lock) | |
1da177e4 | 126 | { |
b7b40ade | 127 | return __spin_trylock_bh(lock); |
1da177e4 | 128 | } |
b7b40ade | 129 | EXPORT_SYMBOL(_spin_trylock_bh); |
892a7c67 | 130 | #endif |
1da177e4 | 131 | |
b7b40ade TG |
132 | #ifndef CONFIG_INLINE_SPIN_LOCK |
133 | void __lockfunc _spin_lock(spinlock_t *lock) | |
1da177e4 | 134 | { |
b7b40ade | 135 | __spin_lock(lock); |
1da177e4 | 136 | } |
b7b40ade | 137 | EXPORT_SYMBOL(_spin_lock); |
892a7c67 | 138 | #endif |
1da177e4 | 139 | |
6beb0009 | 140 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE |
1da177e4 LT |
141 | unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock) |
142 | { | |
69d0ee73 | 143 | return __spin_lock_irqsave(lock); |
1da177e4 LT |
144 | } |
145 | EXPORT_SYMBOL(_spin_lock_irqsave); | |
892a7c67 | 146 | #endif |
1da177e4 | 147 | |
6beb0009 | 148 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ |
1da177e4 LT |
149 | void __lockfunc _spin_lock_irq(spinlock_t *lock) |
150 | { | |
69d0ee73 | 151 | __spin_lock_irq(lock); |
1da177e4 LT |
152 | } |
153 | EXPORT_SYMBOL(_spin_lock_irq); | |
892a7c67 | 154 | #endif |
1da177e4 | 155 | |
6beb0009 | 156 | #ifndef CONFIG_INLINE_SPIN_LOCK_BH |
1da177e4 LT |
157 | void __lockfunc _spin_lock_bh(spinlock_t *lock) |
158 | { | |
69d0ee73 | 159 | __spin_lock_bh(lock); |
1da177e4 LT |
160 | } |
161 | EXPORT_SYMBOL(_spin_lock_bh); | |
892a7c67 | 162 | #endif |
1da177e4 | 163 | |
b7b40ade TG |
164 | #ifndef CONFIG_INLINE_SPIN_UNLOCK |
165 | void __lockfunc _spin_unlock(spinlock_t *lock) | |
1da177e4 | 166 | { |
b7b40ade | 167 | __spin_unlock(lock); |
1da177e4 | 168 | } |
b7b40ade | 169 | EXPORT_SYMBOL(_spin_unlock); |
892a7c67 | 170 | #endif |
1da177e4 | 171 | |
b7b40ade TG |
172 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE |
173 | void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) | |
1da177e4 | 174 | { |
b7b40ade | 175 | __spin_unlock_irqrestore(lock, flags); |
1da177e4 | 176 | } |
b7b40ade | 177 | EXPORT_SYMBOL(_spin_unlock_irqrestore); |
892a7c67 | 178 | #endif |
1da177e4 | 179 | |
b7b40ade TG |
180 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ |
181 | void __lockfunc _spin_unlock_irq(spinlock_t *lock) | |
1da177e4 | 182 | { |
b7b40ade | 183 | __spin_unlock_irq(lock); |
1da177e4 | 184 | } |
b7b40ade | 185 | EXPORT_SYMBOL(_spin_unlock_irq); |
892a7c67 | 186 | #endif |
1da177e4 | 187 | |
b7b40ade TG |
188 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH |
189 | void __lockfunc _spin_unlock_bh(spinlock_t *lock) | |
1da177e4 | 190 | { |
b7b40ade | 191 | __spin_unlock_bh(lock); |
1da177e4 | 192 | } |
b7b40ade | 193 | EXPORT_SYMBOL(_spin_unlock_bh); |
892a7c67 | 194 | #endif |
1da177e4 | 195 | |
b7b40ade TG |
196 | #ifndef CONFIG_INLINE_READ_TRYLOCK |
197 | int __lockfunc _read_trylock(rwlock_t *lock) | |
1da177e4 | 198 | { |
b7b40ade | 199 | return __read_trylock(lock); |
1da177e4 | 200 | } |
b7b40ade | 201 | EXPORT_SYMBOL(_read_trylock); |
892a7c67 | 202 | #endif |
1da177e4 | 203 | |
b7b40ade TG |
204 | #ifndef CONFIG_INLINE_READ_LOCK |
205 | void __lockfunc _read_lock(rwlock_t *lock) | |
1da177e4 | 206 | { |
b7b40ade | 207 | __read_lock(lock); |
1da177e4 | 208 | } |
b7b40ade | 209 | EXPORT_SYMBOL(_read_lock); |
892a7c67 | 210 | #endif |
1da177e4 | 211 | |
b7b40ade TG |
212 | #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE |
213 | unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock) | |
1da177e4 | 214 | { |
b7b40ade | 215 | return __read_lock_irqsave(lock); |
1da177e4 | 216 | } |
b7b40ade | 217 | EXPORT_SYMBOL(_read_lock_irqsave); |
892a7c67 | 218 | #endif |
1da177e4 | 219 | |
b7b40ade TG |
220 | #ifndef CONFIG_INLINE_READ_LOCK_IRQ |
221 | void __lockfunc _read_lock_irq(rwlock_t *lock) | |
1da177e4 | 222 | { |
b7b40ade | 223 | __read_lock_irq(lock); |
1da177e4 | 224 | } |
b7b40ade | 225 | EXPORT_SYMBOL(_read_lock_irq); |
892a7c67 | 226 | #endif |
1da177e4 | 227 | |
b7b40ade TG |
228 | #ifndef CONFIG_INLINE_READ_LOCK_BH |
229 | void __lockfunc _read_lock_bh(rwlock_t *lock) | |
1da177e4 | 230 | { |
b7b40ade | 231 | __read_lock_bh(lock); |
1da177e4 | 232 | } |
b7b40ade | 233 | EXPORT_SYMBOL(_read_lock_bh); |
892a7c67 | 234 | #endif |
1da177e4 | 235 | |
6beb0009 | 236 | #ifndef CONFIG_INLINE_READ_UNLOCK |
1da177e4 LT |
237 | void __lockfunc _read_unlock(rwlock_t *lock) |
238 | { | |
69d0ee73 | 239 | __read_unlock(lock); |
1da177e4 LT |
240 | } |
241 | EXPORT_SYMBOL(_read_unlock); | |
892a7c67 | 242 | #endif |
1da177e4 | 243 | |
6beb0009 | 244 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE |
1da177e4 LT |
245 | void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) |
246 | { | |
69d0ee73 | 247 | __read_unlock_irqrestore(lock, flags); |
1da177e4 LT |
248 | } |
249 | EXPORT_SYMBOL(_read_unlock_irqrestore); | |
892a7c67 | 250 | #endif |
1da177e4 | 251 | |
6beb0009 | 252 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ |
1da177e4 LT |
253 | void __lockfunc _read_unlock_irq(rwlock_t *lock) |
254 | { | |
69d0ee73 | 255 | __read_unlock_irq(lock); |
1da177e4 LT |
256 | } |
257 | EXPORT_SYMBOL(_read_unlock_irq); | |
892a7c67 | 258 | #endif |
1da177e4 | 259 | |
6beb0009 | 260 | #ifndef CONFIG_INLINE_READ_UNLOCK_BH |
1da177e4 LT |
261 | void __lockfunc _read_unlock_bh(rwlock_t *lock) |
262 | { | |
69d0ee73 | 263 | __read_unlock_bh(lock); |
1da177e4 LT |
264 | } |
265 | EXPORT_SYMBOL(_read_unlock_bh); | |
892a7c67 | 266 | #endif |
1da177e4 | 267 | |
b7b40ade TG |
268 | #ifndef CONFIG_INLINE_WRITE_TRYLOCK |
269 | int __lockfunc _write_trylock(rwlock_t *lock) | |
270 | { | |
271 | return __write_trylock(lock); | |
272 | } | |
273 | EXPORT_SYMBOL(_write_trylock); | |
274 | #endif | |
275 | ||
276 | #ifndef CONFIG_INLINE_WRITE_LOCK | |
277 | void __lockfunc _write_lock(rwlock_t *lock) | |
278 | { | |
279 | __write_lock(lock); | |
280 | } | |
281 | EXPORT_SYMBOL(_write_lock); | |
282 | #endif | |
283 | ||
284 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE | |
285 | unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock) | |
286 | { | |
287 | return __write_lock_irqsave(lock); | |
288 | } | |
289 | EXPORT_SYMBOL(_write_lock_irqsave); | |
290 | #endif | |
291 | ||
292 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ | |
293 | void __lockfunc _write_lock_irq(rwlock_t *lock) | |
294 | { | |
295 | __write_lock_irq(lock); | |
296 | } | |
297 | EXPORT_SYMBOL(_write_lock_irq); | |
298 | #endif | |
299 | ||
300 | #ifndef CONFIG_INLINE_WRITE_LOCK_BH | |
301 | void __lockfunc _write_lock_bh(rwlock_t *lock) | |
302 | { | |
303 | __write_lock_bh(lock); | |
304 | } | |
305 | EXPORT_SYMBOL(_write_lock_bh); | |
306 | #endif | |
307 | ||
308 | #ifndef CONFIG_INLINE_WRITE_UNLOCK | |
309 | void __lockfunc _write_unlock(rwlock_t *lock) | |
310 | { | |
311 | __write_unlock(lock); | |
312 | } | |
313 | EXPORT_SYMBOL(_write_unlock); | |
314 | #endif | |
315 | ||
6beb0009 | 316 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE |
1da177e4 LT |
317 | void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) |
318 | { | |
69d0ee73 | 319 | __write_unlock_irqrestore(lock, flags); |
1da177e4 LT |
320 | } |
321 | EXPORT_SYMBOL(_write_unlock_irqrestore); | |
892a7c67 | 322 | #endif |
1da177e4 | 323 | |
6beb0009 | 324 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ |
1da177e4 LT |
325 | void __lockfunc _write_unlock_irq(rwlock_t *lock) |
326 | { | |
69d0ee73 | 327 | __write_unlock_irq(lock); |
1da177e4 LT |
328 | } |
329 | EXPORT_SYMBOL(_write_unlock_irq); | |
892a7c67 | 330 | #endif |
1da177e4 | 331 | |
6beb0009 | 332 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH |
1da177e4 LT |
333 | void __lockfunc _write_unlock_bh(rwlock_t *lock) |
334 | { | |
69d0ee73 | 335 | __write_unlock_bh(lock); |
1da177e4 LT |
336 | } |
337 | EXPORT_SYMBOL(_write_unlock_bh); | |
892a7c67 | 338 | #endif |
1da177e4 | 339 | |
b7b40ade TG |
340 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
341 | ||
342 | void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass) | |
1da177e4 | 343 | { |
b7b40ade TG |
344 | preempt_disable(); |
345 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | |
346 | LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); | |
1da177e4 | 347 | } |
b7b40ade TG |
348 | EXPORT_SYMBOL(_spin_lock_nested); |
349 | ||
350 | unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, | |
351 | int subclass) | |
352 | { | |
353 | unsigned long flags; | |
354 | ||
355 | local_irq_save(flags); | |
356 | preempt_disable(); | |
357 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | |
358 | LOCK_CONTENDED_FLAGS(lock, _raw_spin_trylock, _raw_spin_lock, | |
359 | _raw_spin_lock_flags, &flags); | |
360 | return flags; | |
361 | } | |
362 | EXPORT_SYMBOL(_spin_lock_irqsave_nested); | |
363 | ||
364 | void __lockfunc _spin_lock_nest_lock(spinlock_t *lock, | |
365 | struct lockdep_map *nest_lock) | |
366 | { | |
367 | preempt_disable(); | |
368 | spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); | |
369 | LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); | |
370 | } | |
371 | EXPORT_SYMBOL(_spin_lock_nest_lock); | |
372 | ||
892a7c67 | 373 | #endif |
1da177e4 | 374 | |
0764d23c | 375 | notrace int in_lock_functions(unsigned long addr) |
1da177e4 LT |
376 | { |
377 | /* Linker adds these: start and end of __lockfunc functions */ | |
378 | extern char __lock_text_start[], __lock_text_end[]; | |
379 | ||
380 | return addr >= (unsigned long)__lock_text_start | |
381 | && addr < (unsigned long)__lock_text_end; | |
382 | } | |
383 | EXPORT_SYMBOL(in_lock_functions); |