Commit | Line | Data |
---|---|---|
951f22d5 | 1 | /* |
951f22d5 MS |
2 | * Out of line spinlock code. |
3 | * | |
a53c8fab | 4 | * Copyright IBM Corp. 2004, 2006 |
951f22d5 MS |
5 | * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) |
6 | */ | |
7 | ||
8 | #include <linux/types.h> | |
9 | #include <linux/module.h> | |
10 | #include <linux/spinlock.h> | |
11 | #include <linux/init.h> | |
8b646bd7 | 12 | #include <linux/smp.h> |
951f22d5 MS |
13 | #include <asm/io.h> |
14 | ||
2c72a44e MS |
15 | int spin_retry = -1; |
16 | ||
17 | static int __init spin_retry_init(void) | |
18 | { | |
19 | if (spin_retry < 0) | |
20 | spin_retry = MACHINE_HAS_CAD ? 10 : 1000; | |
21 | return 0; | |
22 | } | |
23 | early_initcall(spin_retry_init); | |
951f22d5 MS |
24 | |
25 | /** | |
26 | * spin_retry= parameter | |
27 | */ | |
28 | static int __init spin_retry_setup(char *str) | |
29 | { | |
30 | spin_retry = simple_strtoul(str, &str, 0); | |
31 | return 1; | |
32 | } | |
33 | __setup("spin_retry=", spin_retry_setup); | |
34 | ||
2c72a44e MS |
35 | static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old) |
36 | { | |
37 | asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock)); | |
38 | } | |
39 | ||
419123f9 MS |
40 | static inline int cpu_is_preempted(int cpu) |
41 | { | |
42 | if (test_cpu_flag_of(CIF_ENABLED_WAIT, cpu)) | |
43 | return 0; | |
44 | if (smp_vcpu_scheduled(cpu)) | |
45 | return 0; | |
46 | return 1; | |
47 | } | |
48 | ||
0199c4e6 | 49 | void arch_spin_lock_wait(arch_spinlock_t *lp) |
951f22d5 | 50 | { |
6c8cd5bb | 51 | unsigned int cpu = SPINLOCK_LOCKVAL; |
59b69787 | 52 | unsigned int owner; |
db1c4515 | 53 | int count, first_diag; |
951f22d5 | 54 | |
db1c4515 | 55 | first_diag = 1; |
951f22d5 | 56 | while (1) { |
470ada6b MS |
57 | owner = ACCESS_ONCE(lp->lock); |
58 | /* Try to get the lock if it is free. */ | |
59 | if (!owner) { | |
60 | if (_raw_compare_and_swap(&lp->lock, 0, cpu)) | |
61 | return; | |
62 | continue; | |
951f22d5 | 63 | } |
db1c4515 | 64 | /* First iteration: check if the lock owner is running. */ |
419123f9 | 65 | if (first_diag && cpu_is_preempted(~owner)) { |
470ada6b | 66 | smp_yield_cpu(~owner); |
db1c4515 | 67 | first_diag = 0; |
470ada6b MS |
68 | continue; |
69 | } | |
70 | /* Loop for a while on the lock value. */ | |
71 | count = spin_retry; | |
72 | do { | |
2c72a44e MS |
73 | if (MACHINE_HAS_CAD) |
74 | _raw_compare_and_delay(&lp->lock, owner); | |
470ada6b MS |
75 | owner = ACCESS_ONCE(lp->lock); |
76 | } while (owner && count-- > 0); | |
77 | if (!owner) | |
78 | continue; | |
79 | /* | |
80 | * For multiple layers of hypervisors, e.g. z/VM + LPAR | |
db1c4515 MS |
81 | * yield the CPU unconditionally. For LPAR rely on the |
82 | * sense running status. | |
470ada6b | 83 | */ |
419123f9 | 84 | if (!MACHINE_IS_LPAR || cpu_is_preempted(~owner)) { |
8b646bd7 | 85 | smp_yield_cpu(~owner); |
db1c4515 MS |
86 | first_diag = 0; |
87 | } | |
951f22d5 MS |
88 | } |
89 | } | |
0199c4e6 | 90 | EXPORT_SYMBOL(arch_spin_lock_wait); |
951f22d5 | 91 | |
0199c4e6 | 92 | void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) |
894cdde2 | 93 | { |
6c8cd5bb | 94 | unsigned int cpu = SPINLOCK_LOCKVAL; |
59b69787 | 95 | unsigned int owner; |
db1c4515 | 96 | int count, first_diag; |
894cdde2 HH |
97 | |
98 | local_irq_restore(flags); | |
db1c4515 | 99 | first_diag = 1; |
894cdde2 | 100 | while (1) { |
470ada6b MS |
101 | owner = ACCESS_ONCE(lp->lock); |
102 | /* Try to get the lock if it is free. */ | |
103 | if (!owner) { | |
104 | local_irq_disable(); | |
105 | if (_raw_compare_and_swap(&lp->lock, 0, cpu)) | |
106 | return; | |
107 | local_irq_restore(flags); | |
84976952 | 108 | continue; |
470ada6b MS |
109 | } |
110 | /* Check if the lock owner is running. */ | |
419123f9 | 111 | if (first_diag && cpu_is_preempted(~owner)) { |
470ada6b | 112 | smp_yield_cpu(~owner); |
db1c4515 | 113 | first_diag = 0; |
470ada6b | 114 | continue; |
894cdde2 | 115 | } |
470ada6b MS |
116 | /* Loop for a while on the lock value. */ |
117 | count = spin_retry; | |
118 | do { | |
2c72a44e MS |
119 | if (MACHINE_HAS_CAD) |
120 | _raw_compare_and_delay(&lp->lock, owner); | |
470ada6b MS |
121 | owner = ACCESS_ONCE(lp->lock); |
122 | } while (owner && count-- > 0); | |
123 | if (!owner) | |
124 | continue; | |
125 | /* | |
126 | * For multiple layers of hypervisors, e.g. z/VM + LPAR | |
db1c4515 MS |
127 | * yield the CPU unconditionally. For LPAR rely on the |
128 | * sense running status. | |
470ada6b | 129 | */ |
419123f9 | 130 | if (!MACHINE_IS_LPAR || cpu_is_preempted(~owner)) { |
8b646bd7 | 131 | smp_yield_cpu(~owner); |
db1c4515 MS |
132 | first_diag = 0; |
133 | } | |
894cdde2 HH |
134 | } |
135 | } | |
0199c4e6 | 136 | EXPORT_SYMBOL(arch_spin_lock_wait_flags); |
894cdde2 | 137 | |
0199c4e6 | 138 | int arch_spin_trylock_retry(arch_spinlock_t *lp) |
951f22d5 | 139 | { |
2c72a44e MS |
140 | unsigned int cpu = SPINLOCK_LOCKVAL; |
141 | unsigned int owner; | |
3c1fcfe2 | 142 | int count; |
951f22d5 | 143 | |
2c72a44e MS |
144 | for (count = spin_retry; count > 0; count--) { |
145 | owner = ACCESS_ONCE(lp->lock); | |
146 | /* Try to get the lock if it is free. */ | |
147 | if (!owner) { | |
148 | if (_raw_compare_and_swap(&lp->lock, 0, cpu)) | |
149 | return 1; | |
150 | } else if (MACHINE_HAS_CAD) | |
151 | _raw_compare_and_delay(&lp->lock, owner); | |
152 | } | |
951f22d5 MS |
153 | return 0; |
154 | } | |
0199c4e6 | 155 | EXPORT_SYMBOL(arch_spin_trylock_retry); |
951f22d5 | 156 | |
fb3a6bbc | 157 | void _raw_read_lock_wait(arch_rwlock_t *rw) |
951f22d5 | 158 | { |
d59b93da | 159 | unsigned int owner, old; |
951f22d5 MS |
160 | int count = spin_retry; |
161 | ||
bbae71bf MS |
162 | #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES |
163 | __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD); | |
164 | #endif | |
d59b93da | 165 | owner = 0; |
951f22d5 MS |
166 | while (1) { |
167 | if (count-- <= 0) { | |
419123f9 | 168 | if (owner && cpu_is_preempted(~owner)) |
d59b93da | 169 | smp_yield_cpu(~owner); |
951f22d5 MS |
170 | count = spin_retry; |
171 | } | |
bae8f567 | 172 | old = ACCESS_ONCE(rw->lock); |
d59b93da | 173 | owner = ACCESS_ONCE(rw->owner); |
2c72a44e MS |
174 | if ((int) old < 0) { |
175 | if (MACHINE_HAS_CAD) | |
176 | _raw_compare_and_delay(&rw->lock, old); | |
96567161 | 177 | continue; |
2c72a44e | 178 | } |
5b3f683e | 179 | if (_raw_compare_and_swap(&rw->lock, old, old + 1)) |
951f22d5 MS |
180 | return; |
181 | } | |
182 | } | |
183 | EXPORT_SYMBOL(_raw_read_lock_wait); | |
184 | ||
fb3a6bbc | 185 | int _raw_read_trylock_retry(arch_rwlock_t *rw) |
951f22d5 MS |
186 | { |
187 | unsigned int old; | |
188 | int count = spin_retry; | |
189 | ||
190 | while (count-- > 0) { | |
bae8f567 | 191 | old = ACCESS_ONCE(rw->lock); |
2c72a44e MS |
192 | if ((int) old < 0) { |
193 | if (MACHINE_HAS_CAD) | |
194 | _raw_compare_and_delay(&rw->lock, old); | |
96567161 | 195 | continue; |
2c72a44e | 196 | } |
5b3f683e | 197 | if (_raw_compare_and_swap(&rw->lock, old, old + 1)) |
951f22d5 MS |
198 | return 1; |
199 | } | |
200 | return 0; | |
201 | } | |
202 | EXPORT_SYMBOL(_raw_read_trylock_retry); | |
203 | ||
bbae71bf MS |
204 | #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES |
205 | ||
206 | void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev) | |
207 | { | |
208 | unsigned int owner, old; | |
209 | int count = spin_retry; | |
210 | ||
211 | owner = 0; | |
212 | while (1) { | |
213 | if (count-- <= 0) { | |
419123f9 | 214 | if (owner && cpu_is_preempted(~owner)) |
bbae71bf MS |
215 | smp_yield_cpu(~owner); |
216 | count = spin_retry; | |
217 | } | |
218 | old = ACCESS_ONCE(rw->lock); | |
219 | owner = ACCESS_ONCE(rw->owner); | |
e0af21c5 | 220 | smp_mb(); |
bbae71bf MS |
221 | if ((int) old >= 0) { |
222 | prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR); | |
223 | old = prev; | |
224 | } | |
225 | if ((old & 0x7fffffff) == 0 && (int) prev >= 0) | |
226 | break; | |
2c72a44e MS |
227 | if (MACHINE_HAS_CAD) |
228 | _raw_compare_and_delay(&rw->lock, old); | |
bbae71bf MS |
229 | } |
230 | } | |
231 | EXPORT_SYMBOL(_raw_write_lock_wait); | |
232 | ||
233 | #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ | |
234 | ||
fb3a6bbc | 235 | void _raw_write_lock_wait(arch_rwlock_t *rw) |
951f22d5 | 236 | { |
94232a43 | 237 | unsigned int owner, old, prev; |
951f22d5 MS |
238 | int count = spin_retry; |
239 | ||
94232a43 | 240 | prev = 0x80000000; |
d59b93da | 241 | owner = 0; |
951f22d5 MS |
242 | while (1) { |
243 | if (count-- <= 0) { | |
419123f9 | 244 | if (owner && cpu_is_preempted(~owner)) |
d59b93da | 245 | smp_yield_cpu(~owner); |
951f22d5 MS |
246 | count = spin_retry; |
247 | } | |
bae8f567 | 248 | old = ACCESS_ONCE(rw->lock); |
d59b93da | 249 | owner = ACCESS_ONCE(rw->owner); |
94232a43 MS |
250 | if ((int) old >= 0 && |
251 | _raw_compare_and_swap(&rw->lock, old, old | 0x80000000)) | |
252 | prev = old; | |
253 | else | |
e0af21c5 | 254 | smp_mb(); |
94232a43 MS |
255 | if ((old & 0x7fffffff) == 0 && (int) prev >= 0) |
256 | break; | |
2c72a44e MS |
257 | if (MACHINE_HAS_CAD) |
258 | _raw_compare_and_delay(&rw->lock, old); | |
951f22d5 MS |
259 | } |
260 | } | |
261 | EXPORT_SYMBOL(_raw_write_lock_wait); | |
262 | ||
bbae71bf MS |
263 | #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ |
264 | ||
fb3a6bbc | 265 | int _raw_write_trylock_retry(arch_rwlock_t *rw) |
951f22d5 | 266 | { |
bae8f567 | 267 | unsigned int old; |
951f22d5 MS |
268 | int count = spin_retry; |
269 | ||
270 | while (count-- > 0) { | |
bae8f567 | 271 | old = ACCESS_ONCE(rw->lock); |
2c72a44e MS |
272 | if (old) { |
273 | if (MACHINE_HAS_CAD) | |
274 | _raw_compare_and_delay(&rw->lock, old); | |
96567161 | 275 | continue; |
2c72a44e | 276 | } |
5b3f683e | 277 | if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000)) |
951f22d5 MS |
278 | return 1; |
279 | } | |
280 | return 0; | |
281 | } | |
282 | EXPORT_SYMBOL(_raw_write_trylock_retry); | |
d59b93da MS |
283 | |
284 | void arch_lock_relax(unsigned int cpu) | |
285 | { | |
286 | if (!cpu) | |
287 | return; | |
419123f9 | 288 | if (MACHINE_IS_LPAR && !cpu_is_preempted(~cpu)) |
d59b93da MS |
289 | return; |
290 | smp_yield_cpu(~cpu); | |
291 | } | |
292 | EXPORT_SYMBOL(arch_lock_relax); |