Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Atomic operations that C can't guarantee us. Useful for | |
3 | * resource counting etc.. | |
4 | * | |
5 | * But use these as seldom as possible since they are much more slower | |
6 | * than regular operations. | |
7 | * | |
8 | * This file is subject to the terms and conditions of the GNU General Public | |
9 | * License. See the file "COPYING" in the main directory of this archive | |
10 | * for more details. | |
11 | * | |
e303e088 | 12 | * Copyright (C) 1996, 97, 99, 2000, 03, 04, 06 by Ralf Baechle |
1da177e4 | 13 | */ |
1da177e4 LT |
14 | #ifndef _ASM_ATOMIC_H |
15 | #define _ASM_ATOMIC_H | |
16 | ||
192ef366 | 17 | #include <linux/irqflags.h> |
ea435467 | 18 | #include <linux/types.h> |
0004a9df | 19 | #include <asm/barrier.h> |
1da177e4 | 20 | #include <asm/cpu-features.h> |
b81947c6 | 21 | #include <asm/cmpxchg.h> |
1da177e4 LT |
22 | #include <asm/war.h> |
23 | ||
1da177e4 LT |
24 | #define ATOMIC_INIT(i) { (i) } |
25 | ||
26 | /* | |
27 | * atomic_read - read atomic variable | |
28 | * @v: pointer of type atomic_t | |
29 | * | |
30 | * Atomically reads the value of @v. | |
31 | */ | |
f3d46f9d | 32 | #define atomic_read(v) (*(volatile int *)&(v)->counter) |
1da177e4 LT |
33 | |
34 | /* | |
35 | * atomic_set - set atomic variable | |
36 | * @v: pointer of type atomic_t | |
37 | * @i: required value | |
38 | * | |
39 | * Atomically sets the value of @v to @i. | |
40 | */ | |
21a151d8 | 41 | #define atomic_set(v, i) ((v)->counter = (i)) |
1da177e4 LT |
42 | |
43 | /* | |
44 | * atomic_add - add integer to atomic variable | |
45 | * @i: integer value to add | |
46 | * @v: pointer of type atomic_t | |
47 | * | |
48 | * Atomically adds @i to @v. | |
49 | */ | |
50 | static __inline__ void atomic_add(int i, atomic_t * v) | |
51 | { | |
b791d119 | 52 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
915ec1e2 | 53 | int temp; |
1da177e4 LT |
54 | |
55 | __asm__ __volatile__( | |
c4559f67 | 56 | " .set mips3 \n" |
1da177e4 LT |
57 | "1: ll %0, %1 # atomic_add \n" |
58 | " addu %0, %2 \n" | |
59 | " sc %0, %1 \n" | |
60 | " beqzl %0, 1b \n" | |
aac8aa77 | 61 | " .set mips0 \n" |
1da177e4 LT |
62 | : "=&r" (temp), "=m" (v->counter) |
63 | : "Ir" (i), "m" (v->counter)); | |
b791d119 | 64 | } else if (kernel_uses_llsc) { |
915ec1e2 | 65 | int temp; |
1da177e4 | 66 | |
7837314d RB |
67 | do { |
68 | __asm__ __volatile__( | |
69 | " .set mips3 \n" | |
70 | " ll %0, %1 # atomic_add \n" | |
71 | " addu %0, %2 \n" | |
72 | " sc %0, %1 \n" | |
73 | " .set mips0 \n" | |
74 | : "=&r" (temp), "=m" (v->counter) | |
75 | : "Ir" (i), "m" (v->counter)); | |
76 | } while (unlikely(!temp)); | |
1da177e4 LT |
77 | } else { |
78 | unsigned long flags; | |
79 | ||
49edd098 | 80 | raw_local_irq_save(flags); |
1da177e4 | 81 | v->counter += i; |
49edd098 | 82 | raw_local_irq_restore(flags); |
1da177e4 LT |
83 | } |
84 | } | |
85 | ||
86 | /* | |
87 | * atomic_sub - subtract the atomic variable | |
88 | * @i: integer value to subtract | |
89 | * @v: pointer of type atomic_t | |
90 | * | |
91 | * Atomically subtracts @i from @v. | |
92 | */ | |
93 | static __inline__ void atomic_sub(int i, atomic_t * v) | |
94 | { | |
b791d119 | 95 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
915ec1e2 | 96 | int temp; |
1da177e4 LT |
97 | |
98 | __asm__ __volatile__( | |
c4559f67 | 99 | " .set mips3 \n" |
1da177e4 LT |
100 | "1: ll %0, %1 # atomic_sub \n" |
101 | " subu %0, %2 \n" | |
102 | " sc %0, %1 \n" | |
103 | " beqzl %0, 1b \n" | |
aac8aa77 | 104 | " .set mips0 \n" |
1da177e4 LT |
105 | : "=&r" (temp), "=m" (v->counter) |
106 | : "Ir" (i), "m" (v->counter)); | |
b791d119 | 107 | } else if (kernel_uses_llsc) { |
915ec1e2 | 108 | int temp; |
1da177e4 | 109 | |
7837314d RB |
110 | do { |
111 | __asm__ __volatile__( | |
112 | " .set mips3 \n" | |
113 | " ll %0, %1 # atomic_sub \n" | |
114 | " subu %0, %2 \n" | |
115 | " sc %0, %1 \n" | |
116 | " .set mips0 \n" | |
117 | : "=&r" (temp), "=m" (v->counter) | |
118 | : "Ir" (i), "m" (v->counter)); | |
119 | } while (unlikely(!temp)); | |
1da177e4 LT |
120 | } else { |
121 | unsigned long flags; | |
122 | ||
49edd098 | 123 | raw_local_irq_save(flags); |
1da177e4 | 124 | v->counter -= i; |
49edd098 | 125 | raw_local_irq_restore(flags); |
1da177e4 LT |
126 | } |
127 | } | |
128 | ||
129 | /* | |
130 | * Same as above, but return the result value | |
131 | */ | |
132 | static __inline__ int atomic_add_return(int i, atomic_t * v) | |
133 | { | |
915ec1e2 | 134 | int result; |
1da177e4 | 135 | |
f252ffd5 | 136 | smp_mb__before_llsc(); |
0004a9df | 137 | |
b791d119 | 138 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
915ec1e2 | 139 | int temp; |
1da177e4 LT |
140 | |
141 | __asm__ __volatile__( | |
c4559f67 | 142 | " .set mips3 \n" |
1da177e4 LT |
143 | "1: ll %1, %2 # atomic_add_return \n" |
144 | " addu %0, %1, %3 \n" | |
145 | " sc %0, %2 \n" | |
146 | " beqzl %0, 1b \n" | |
147 | " addu %0, %1, %3 \n" | |
aac8aa77 | 148 | " .set mips0 \n" |
1da177e4 LT |
149 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) |
150 | : "Ir" (i), "m" (v->counter) | |
151 | : "memory"); | |
b791d119 | 152 | } else if (kernel_uses_llsc) { |
915ec1e2 | 153 | int temp; |
1da177e4 | 154 | |
7837314d RB |
155 | do { |
156 | __asm__ __volatile__( | |
157 | " .set mips3 \n" | |
158 | " ll %1, %2 # atomic_add_return \n" | |
159 | " addu %0, %1, %3 \n" | |
160 | " sc %0, %2 \n" | |
161 | " .set mips0 \n" | |
162 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) | |
163 | : "Ir" (i), "m" (v->counter) | |
164 | : "memory"); | |
165 | } while (unlikely(!result)); | |
166 | ||
167 | result = temp + i; | |
1da177e4 LT |
168 | } else { |
169 | unsigned long flags; | |
170 | ||
49edd098 | 171 | raw_local_irq_save(flags); |
1da177e4 LT |
172 | result = v->counter; |
173 | result += i; | |
174 | v->counter = result; | |
49edd098 | 175 | raw_local_irq_restore(flags); |
1da177e4 LT |
176 | } |
177 | ||
17099b11 | 178 | smp_llsc_mb(); |
0004a9df | 179 | |
1da177e4 LT |
180 | return result; |
181 | } | |
182 | ||
183 | static __inline__ int atomic_sub_return(int i, atomic_t * v) | |
184 | { | |
915ec1e2 | 185 | int result; |
1da177e4 | 186 | |
f252ffd5 | 187 | smp_mb__before_llsc(); |
0004a9df | 188 | |
b791d119 | 189 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
915ec1e2 | 190 | int temp; |
1da177e4 LT |
191 | |
192 | __asm__ __volatile__( | |
c4559f67 | 193 | " .set mips3 \n" |
1da177e4 LT |
194 | "1: ll %1, %2 # atomic_sub_return \n" |
195 | " subu %0, %1, %3 \n" | |
196 | " sc %0, %2 \n" | |
197 | " beqzl %0, 1b \n" | |
198 | " subu %0, %1, %3 \n" | |
aac8aa77 | 199 | " .set mips0 \n" |
1da177e4 LT |
200 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) |
201 | : "Ir" (i), "m" (v->counter) | |
202 | : "memory"); | |
7837314d RB |
203 | |
204 | result = temp - i; | |
b791d119 | 205 | } else if (kernel_uses_llsc) { |
915ec1e2 | 206 | int temp; |
1da177e4 | 207 | |
7837314d RB |
208 | do { |
209 | __asm__ __volatile__( | |
210 | " .set mips3 \n" | |
211 | " ll %1, %2 # atomic_sub_return \n" | |
212 | " subu %0, %1, %3 \n" | |
213 | " sc %0, %2 \n" | |
214 | " .set mips0 \n" | |
215 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) | |
216 | : "Ir" (i), "m" (v->counter) | |
217 | : "memory"); | |
218 | } while (unlikely(!result)); | |
219 | ||
220 | result = temp - i; | |
1da177e4 LT |
221 | } else { |
222 | unsigned long flags; | |
223 | ||
49edd098 | 224 | raw_local_irq_save(flags); |
1da177e4 LT |
225 | result = v->counter; |
226 | result -= i; | |
227 | v->counter = result; | |
49edd098 | 228 | raw_local_irq_restore(flags); |
1da177e4 LT |
229 | } |
230 | ||
17099b11 | 231 | smp_llsc_mb(); |
0004a9df | 232 | |
1da177e4 LT |
233 | return result; |
234 | } | |
235 | ||
236 | /* | |
f10d14dd AG |
237 | * atomic_sub_if_positive - conditionally subtract integer from atomic variable |
238 | * @i: integer value to subtract | |
1da177e4 LT |
239 | * @v: pointer of type atomic_t |
240 | * | |
f10d14dd AG |
241 | * Atomically test @v and subtract @i if @v is greater or equal than @i. |
242 | * The function returns the old value of @v minus @i. | |
1da177e4 LT |
243 | */ |
244 | static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) | |
245 | { | |
915ec1e2 | 246 | int result; |
1da177e4 | 247 | |
f252ffd5 | 248 | smp_mb__before_llsc(); |
0004a9df | 249 | |
b791d119 | 250 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
915ec1e2 | 251 | int temp; |
1da177e4 LT |
252 | |
253 | __asm__ __volatile__( | |
c4559f67 | 254 | " .set mips3 \n" |
1da177e4 LT |
255 | "1: ll %1, %2 # atomic_sub_if_positive\n" |
256 | " subu %0, %1, %3 \n" | |
257 | " bltz %0, 1f \n" | |
258 | " sc %0, %2 \n" | |
92f22c18 | 259 | " .set noreorder \n" |
1da177e4 | 260 | " beqzl %0, 1b \n" |
92f22c18 RB |
261 | " subu %0, %1, %3 \n" |
262 | " .set reorder \n" | |
1da177e4 | 263 | "1: \n" |
aac8aa77 | 264 | " .set mips0 \n" |
1da177e4 LT |
265 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) |
266 | : "Ir" (i), "m" (v->counter) | |
267 | : "memory"); | |
b791d119 | 268 | } else if (kernel_uses_llsc) { |
915ec1e2 | 269 | int temp; |
1da177e4 LT |
270 | |
271 | __asm__ __volatile__( | |
c4559f67 | 272 | " .set mips3 \n" |
1da177e4 LT |
273 | "1: ll %1, %2 # atomic_sub_if_positive\n" |
274 | " subu %0, %1, %3 \n" | |
275 | " bltz %0, 1f \n" | |
276 | " sc %0, %2 \n" | |
92f22c18 | 277 | " .set noreorder \n" |
7837314d | 278 | " beqz %0, 1b \n" |
92f22c18 RB |
279 | " subu %0, %1, %3 \n" |
280 | " .set reorder \n" | |
50952026 | 281 | "1: \n" |
aac8aa77 | 282 | " .set mips0 \n" |
1da177e4 LT |
283 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) |
284 | : "Ir" (i), "m" (v->counter) | |
285 | : "memory"); | |
286 | } else { | |
287 | unsigned long flags; | |
288 | ||
49edd098 | 289 | raw_local_irq_save(flags); |
1da177e4 LT |
290 | result = v->counter; |
291 | result -= i; | |
292 | if (result >= 0) | |
293 | v->counter = result; | |
49edd098 | 294 | raw_local_irq_restore(flags); |
1da177e4 LT |
295 | } |
296 | ||
17099b11 | 297 | smp_llsc_mb(); |
0004a9df | 298 | |
1da177e4 LT |
299 | return result; |
300 | } | |
301 | ||
e12f644b MD |
302 | #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) |
303 | #define atomic_xchg(v, new) (xchg(&((v)->counter), (new))) | |
4a6dae6d | 304 | |
8426e1f6 | 305 | /** |
f24219b4 | 306 | * __atomic_add_unless - add unless the number is a given value |
8426e1f6 NP |
307 | * @v: pointer of type atomic_t |
308 | * @a: the amount to add to v... | |
309 | * @u: ...unless v is equal to u. | |
310 | * | |
311 | * Atomically adds @a to @v, so long as it was not @u. | |
f24219b4 | 312 | * Returns the old value of @v. |
8426e1f6 | 313 | */ |
f24219b4 | 314 | static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) |
2856f5e3 MD |
315 | { |
316 | int c, old; | |
317 | c = atomic_read(v); | |
318 | for (;;) { | |
319 | if (unlikely(c == (u))) | |
320 | break; | |
321 | old = atomic_cmpxchg((v), c, c + (a)); | |
322 | if (likely(old == c)) | |
323 | break; | |
324 | c = old; | |
325 | } | |
f24219b4 | 326 | return c; |
2856f5e3 | 327 | } |
8426e1f6 | 328 | |
21a151d8 RB |
329 | #define atomic_dec_return(v) atomic_sub_return(1, (v)) |
330 | #define atomic_inc_return(v) atomic_add_return(1, (v)) | |
1da177e4 LT |
331 | |
332 | /* | |
333 | * atomic_sub_and_test - subtract value from variable and test result | |
334 | * @i: integer value to subtract | |
335 | * @v: pointer of type atomic_t | |
336 | * | |
337 | * Atomically subtracts @i from @v and returns | |
338 | * true if the result is zero, or false for all | |
339 | * other cases. | |
340 | */ | |
21a151d8 | 341 | #define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0) |
1da177e4 LT |
342 | |
343 | /* | |
344 | * atomic_inc_and_test - increment and test | |
345 | * @v: pointer of type atomic_t | |
346 | * | |
347 | * Atomically increments @v by 1 | |
348 | * and returns true if the result is zero, or false for all | |
349 | * other cases. | |
350 | */ | |
351 | #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) | |
352 | ||
353 | /* | |
354 | * atomic_dec_and_test - decrement by 1 and test | |
355 | * @v: pointer of type atomic_t | |
356 | * | |
357 | * Atomically decrements @v by 1 and | |
358 | * returns true if the result is 0, or false for all other | |
359 | * cases. | |
360 | */ | |
361 | #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) | |
362 | ||
363 | /* | |
364 | * atomic_dec_if_positive - decrement by 1 if old value positive | |
365 | * @v: pointer of type atomic_t | |
366 | */ | |
367 | #define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v) | |
368 | ||
369 | /* | |
370 | * atomic_inc - increment atomic variable | |
371 | * @v: pointer of type atomic_t | |
372 | * | |
373 | * Atomically increments @v by 1. | |
374 | */ | |
21a151d8 | 375 | #define atomic_inc(v) atomic_add(1, (v)) |
1da177e4 LT |
376 | |
377 | /* | |
378 | * atomic_dec - decrement and test | |
379 | * @v: pointer of type atomic_t | |
380 | * | |
381 | * Atomically decrements @v by 1. | |
382 | */ | |
21a151d8 | 383 | #define atomic_dec(v) atomic_sub(1, (v)) |
1da177e4 LT |
384 | |
385 | /* | |
386 | * atomic_add_negative - add and test if negative | |
387 | * @v: pointer of type atomic_t | |
388 | * @i: integer value to add | |
389 | * | |
390 | * Atomically adds @i to @v and returns true | |
391 | * if the result is negative, or false when | |
392 | * result is greater than or equal to zero. | |
393 | */ | |
21a151d8 | 394 | #define atomic_add_negative(i, v) (atomic_add_return(i, (v)) < 0) |
1da177e4 | 395 | |
875d43e7 | 396 | #ifdef CONFIG_64BIT |
1da177e4 | 397 | |
1da177e4 LT |
398 | #define ATOMIC64_INIT(i) { (i) } |
399 | ||
400 | /* | |
401 | * atomic64_read - read atomic variable | |
402 | * @v: pointer of type atomic64_t | |
403 | * | |
404 | */ | |
f3d46f9d | 405 | #define atomic64_read(v) (*(volatile long *)&(v)->counter) |
1da177e4 LT |
406 | |
407 | /* | |
408 | * atomic64_set - set atomic variable | |
409 | * @v: pointer of type atomic64_t | |
410 | * @i: required value | |
411 | */ | |
21a151d8 | 412 | #define atomic64_set(v, i) ((v)->counter = (i)) |
1da177e4 LT |
413 | |
414 | /* | |
415 | * atomic64_add - add integer to atomic variable | |
416 | * @i: integer value to add | |
417 | * @v: pointer of type atomic64_t | |
418 | * | |
419 | * Atomically adds @i to @v. | |
420 | */ | |
421 | static __inline__ void atomic64_add(long i, atomic64_t * v) | |
422 | { | |
b791d119 | 423 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
915ec1e2 | 424 | long temp; |
1da177e4 LT |
425 | |
426 | __asm__ __volatile__( | |
aac8aa77 | 427 | " .set mips3 \n" |
1da177e4 | 428 | "1: lld %0, %1 # atomic64_add \n" |
f2a68272 | 429 | " daddu %0, %2 \n" |
1da177e4 LT |
430 | " scd %0, %1 \n" |
431 | " beqzl %0, 1b \n" | |
aac8aa77 | 432 | " .set mips0 \n" |
1da177e4 LT |
433 | : "=&r" (temp), "=m" (v->counter) |
434 | : "Ir" (i), "m" (v->counter)); | |
b791d119 | 435 | } else if (kernel_uses_llsc) { |
915ec1e2 | 436 | long temp; |
1da177e4 | 437 | |
7837314d RB |
438 | do { |
439 | __asm__ __volatile__( | |
440 | " .set mips3 \n" | |
441 | " lld %0, %1 # atomic64_add \n" | |
442 | " daddu %0, %2 \n" | |
443 | " scd %0, %1 \n" | |
444 | " .set mips0 \n" | |
445 | : "=&r" (temp), "=m" (v->counter) | |
446 | : "Ir" (i), "m" (v->counter)); | |
447 | } while (unlikely(!temp)); | |
1da177e4 LT |
448 | } else { |
449 | unsigned long flags; | |
450 | ||
49edd098 | 451 | raw_local_irq_save(flags); |
1da177e4 | 452 | v->counter += i; |
49edd098 | 453 | raw_local_irq_restore(flags); |
1da177e4 LT |
454 | } |
455 | } | |
456 | ||
457 | /* | |
458 | * atomic64_sub - subtract the atomic variable | |
459 | * @i: integer value to subtract | |
460 | * @v: pointer of type atomic64_t | |
461 | * | |
462 | * Atomically subtracts @i from @v. | |
463 | */ | |
464 | static __inline__ void atomic64_sub(long i, atomic64_t * v) | |
465 | { | |
b791d119 | 466 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
915ec1e2 | 467 | long temp; |
1da177e4 LT |
468 | |
469 | __asm__ __volatile__( | |
aac8aa77 | 470 | " .set mips3 \n" |
1da177e4 | 471 | "1: lld %0, %1 # atomic64_sub \n" |
f2a68272 | 472 | " dsubu %0, %2 \n" |
1da177e4 LT |
473 | " scd %0, %1 \n" |
474 | " beqzl %0, 1b \n" | |
aac8aa77 | 475 | " .set mips0 \n" |
1da177e4 LT |
476 | : "=&r" (temp), "=m" (v->counter) |
477 | : "Ir" (i), "m" (v->counter)); | |
b791d119 | 478 | } else if (kernel_uses_llsc) { |
915ec1e2 | 479 | long temp; |
1da177e4 | 480 | |
7837314d RB |
481 | do { |
482 | __asm__ __volatile__( | |
483 | " .set mips3 \n" | |
484 | " lld %0, %1 # atomic64_sub \n" | |
485 | " dsubu %0, %2 \n" | |
486 | " scd %0, %1 \n" | |
487 | " .set mips0 \n" | |
488 | : "=&r" (temp), "=m" (v->counter) | |
489 | : "Ir" (i), "m" (v->counter)); | |
490 | } while (unlikely(!temp)); | |
1da177e4 LT |
491 | } else { |
492 | unsigned long flags; | |
493 | ||
49edd098 | 494 | raw_local_irq_save(flags); |
1da177e4 | 495 | v->counter -= i; |
49edd098 | 496 | raw_local_irq_restore(flags); |
1da177e4 LT |
497 | } |
498 | } | |
499 | ||
500 | /* | |
501 | * Same as above, but return the result value | |
502 | */ | |
503 | static __inline__ long atomic64_add_return(long i, atomic64_t * v) | |
504 | { | |
915ec1e2 | 505 | long result; |
1da177e4 | 506 | |
f252ffd5 | 507 | smp_mb__before_llsc(); |
0004a9df | 508 | |
b791d119 | 509 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
915ec1e2 | 510 | long temp; |
1da177e4 LT |
511 | |
512 | __asm__ __volatile__( | |
aac8aa77 | 513 | " .set mips3 \n" |
1da177e4 | 514 | "1: lld %1, %2 # atomic64_add_return \n" |
f2a68272 | 515 | " daddu %0, %1, %3 \n" |
1da177e4 LT |
516 | " scd %0, %2 \n" |
517 | " beqzl %0, 1b \n" | |
f2a68272 | 518 | " daddu %0, %1, %3 \n" |
aac8aa77 | 519 | " .set mips0 \n" |
1da177e4 LT |
520 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) |
521 | : "Ir" (i), "m" (v->counter) | |
522 | : "memory"); | |
b791d119 | 523 | } else if (kernel_uses_llsc) { |
915ec1e2 | 524 | long temp; |
1da177e4 | 525 | |
7837314d RB |
526 | do { |
527 | __asm__ __volatile__( | |
528 | " .set mips3 \n" | |
529 | " lld %1, %2 # atomic64_add_return \n" | |
530 | " daddu %0, %1, %3 \n" | |
531 | " scd %0, %2 \n" | |
532 | " .set mips0 \n" | |
533 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) | |
534 | : "Ir" (i), "m" (v->counter) | |
535 | : "memory"); | |
536 | } while (unlikely(!result)); | |
537 | ||
538 | result = temp + i; | |
1da177e4 LT |
539 | } else { |
540 | unsigned long flags; | |
541 | ||
49edd098 | 542 | raw_local_irq_save(flags); |
1da177e4 LT |
543 | result = v->counter; |
544 | result += i; | |
545 | v->counter = result; | |
49edd098 | 546 | raw_local_irq_restore(flags); |
1da177e4 LT |
547 | } |
548 | ||
17099b11 | 549 | smp_llsc_mb(); |
0004a9df | 550 | |
1da177e4 LT |
551 | return result; |
552 | } | |
553 | ||
554 | static __inline__ long atomic64_sub_return(long i, atomic64_t * v) | |
555 | { | |
915ec1e2 | 556 | long result; |
1da177e4 | 557 | |
f252ffd5 | 558 | smp_mb__before_llsc(); |
0004a9df | 559 | |
b791d119 | 560 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
915ec1e2 | 561 | long temp; |
1da177e4 LT |
562 | |
563 | __asm__ __volatile__( | |
aac8aa77 | 564 | " .set mips3 \n" |
1da177e4 | 565 | "1: lld %1, %2 # atomic64_sub_return \n" |
f2a68272 | 566 | " dsubu %0, %1, %3 \n" |
1da177e4 LT |
567 | " scd %0, %2 \n" |
568 | " beqzl %0, 1b \n" | |
f2a68272 | 569 | " dsubu %0, %1, %3 \n" |
aac8aa77 | 570 | " .set mips0 \n" |
1da177e4 LT |
571 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) |
572 | : "Ir" (i), "m" (v->counter) | |
573 | : "memory"); | |
b791d119 | 574 | } else if (kernel_uses_llsc) { |
915ec1e2 | 575 | long temp; |
1da177e4 | 576 | |
7837314d RB |
577 | do { |
578 | __asm__ __volatile__( | |
579 | " .set mips3 \n" | |
580 | " lld %1, %2 # atomic64_sub_return \n" | |
581 | " dsubu %0, %1, %3 \n" | |
582 | " scd %0, %2 \n" | |
583 | " .set mips0 \n" | |
584 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) | |
585 | : "Ir" (i), "m" (v->counter) | |
586 | : "memory"); | |
587 | } while (unlikely(!result)); | |
588 | ||
589 | result = temp - i; | |
1da177e4 LT |
590 | } else { |
591 | unsigned long flags; | |
592 | ||
49edd098 | 593 | raw_local_irq_save(flags); |
1da177e4 LT |
594 | result = v->counter; |
595 | result -= i; | |
596 | v->counter = result; | |
49edd098 | 597 | raw_local_irq_restore(flags); |
1da177e4 LT |
598 | } |
599 | ||
17099b11 | 600 | smp_llsc_mb(); |
0004a9df | 601 | |
1da177e4 LT |
602 | return result; |
603 | } | |
604 | ||
605 | /* | |
f10d14dd AG |
606 | * atomic64_sub_if_positive - conditionally subtract integer from atomic variable |
607 | * @i: integer value to subtract | |
1da177e4 LT |
608 | * @v: pointer of type atomic64_t |
609 | * | |
f10d14dd AG |
610 | * Atomically test @v and subtract @i if @v is greater or equal than @i. |
611 | * The function returns the old value of @v minus @i. | |
1da177e4 LT |
612 | */ |
613 | static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) | |
614 | { | |
915ec1e2 | 615 | long result; |
1da177e4 | 616 | |
f252ffd5 | 617 | smp_mb__before_llsc(); |
0004a9df | 618 | |
b791d119 | 619 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
915ec1e2 | 620 | long temp; |
1da177e4 LT |
621 | |
622 | __asm__ __volatile__( | |
aac8aa77 | 623 | " .set mips3 \n" |
1da177e4 LT |
624 | "1: lld %1, %2 # atomic64_sub_if_positive\n" |
625 | " dsubu %0, %1, %3 \n" | |
626 | " bltz %0, 1f \n" | |
627 | " scd %0, %2 \n" | |
92f22c18 | 628 | " .set noreorder \n" |
1da177e4 | 629 | " beqzl %0, 1b \n" |
92f22c18 RB |
630 | " dsubu %0, %1, %3 \n" |
631 | " .set reorder \n" | |
1da177e4 | 632 | "1: \n" |
aac8aa77 | 633 | " .set mips0 \n" |
1da177e4 LT |
634 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) |
635 | : "Ir" (i), "m" (v->counter) | |
636 | : "memory"); | |
b791d119 | 637 | } else if (kernel_uses_llsc) { |
915ec1e2 | 638 | long temp; |
1da177e4 LT |
639 | |
640 | __asm__ __volatile__( | |
aac8aa77 | 641 | " .set mips3 \n" |
1da177e4 LT |
642 | "1: lld %1, %2 # atomic64_sub_if_positive\n" |
643 | " dsubu %0, %1, %3 \n" | |
644 | " bltz %0, 1f \n" | |
645 | " scd %0, %2 \n" | |
92f22c18 | 646 | " .set noreorder \n" |
7837314d | 647 | " beqz %0, 1b \n" |
92f22c18 RB |
648 | " dsubu %0, %1, %3 \n" |
649 | " .set reorder \n" | |
50952026 | 650 | "1: \n" |
aac8aa77 | 651 | " .set mips0 \n" |
1da177e4 LT |
652 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) |
653 | : "Ir" (i), "m" (v->counter) | |
654 | : "memory"); | |
655 | } else { | |
656 | unsigned long flags; | |
657 | ||
49edd098 | 658 | raw_local_irq_save(flags); |
1da177e4 LT |
659 | result = v->counter; |
660 | result -= i; | |
661 | if (result >= 0) | |
662 | v->counter = result; | |
49edd098 | 663 | raw_local_irq_restore(flags); |
1da177e4 LT |
664 | } |
665 | ||
17099b11 | 666 | smp_llsc_mb(); |
0004a9df | 667 | |
1da177e4 LT |
668 | return result; |
669 | } | |
670 | ||
e12f644b | 671 | #define atomic64_cmpxchg(v, o, n) \ |
7b239bb1 | 672 | ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) |
e12f644b MD |
673 | #define atomic64_xchg(v, new) (xchg(&((v)->counter), (new))) |
674 | ||
675 | /** | |
676 | * atomic64_add_unless - add unless the number is a given value | |
677 | * @v: pointer of type atomic64_t | |
678 | * @a: the amount to add to v... | |
679 | * @u: ...unless v is equal to u. | |
680 | * | |
681 | * Atomically adds @a to @v, so long as it was not @u. | |
f24219b4 | 682 | * Returns the old value of @v. |
e12f644b | 683 | */ |
2856f5e3 MD |
684 | static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) |
685 | { | |
686 | long c, old; | |
687 | c = atomic64_read(v); | |
688 | for (;;) { | |
689 | if (unlikely(c == (u))) | |
690 | break; | |
691 | old = atomic64_cmpxchg((v), c, c + (a)); | |
692 | if (likely(old == c)) | |
693 | break; | |
694 | c = old; | |
695 | } | |
696 | return c != (u); | |
697 | } | |
698 | ||
e12f644b MD |
699 | #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) |
700 | ||
21a151d8 RB |
701 | #define atomic64_dec_return(v) atomic64_sub_return(1, (v)) |
702 | #define atomic64_inc_return(v) atomic64_add_return(1, (v)) | |
1da177e4 LT |
703 | |
704 | /* | |
705 | * atomic64_sub_and_test - subtract value from variable and test result | |
706 | * @i: integer value to subtract | |
707 | * @v: pointer of type atomic64_t | |
708 | * | |
709 | * Atomically subtracts @i from @v and returns | |
710 | * true if the result is zero, or false for all | |
711 | * other cases. | |
712 | */ | |
21a151d8 | 713 | #define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0) |
1da177e4 LT |
714 | |
715 | /* | |
716 | * atomic64_inc_and_test - increment and test | |
717 | * @v: pointer of type atomic64_t | |
718 | * | |
719 | * Atomically increments @v by 1 | |
720 | * and returns true if the result is zero, or false for all | |
721 | * other cases. | |
722 | */ | |
723 | #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) | |
724 | ||
725 | /* | |
726 | * atomic64_dec_and_test - decrement by 1 and test | |
727 | * @v: pointer of type atomic64_t | |
728 | * | |
729 | * Atomically decrements @v by 1 and | |
730 | * returns true if the result is 0, or false for all other | |
731 | * cases. | |
732 | */ | |
733 | #define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0) | |
734 | ||
735 | /* | |
736 | * atomic64_dec_if_positive - decrement by 1 if old value positive | |
737 | * @v: pointer of type atomic64_t | |
738 | */ | |
739 | #define atomic64_dec_if_positive(v) atomic64_sub_if_positive(1, v) | |
740 | ||
741 | /* | |
742 | * atomic64_inc - increment atomic variable | |
743 | * @v: pointer of type atomic64_t | |
744 | * | |
745 | * Atomically increments @v by 1. | |
746 | */ | |
21a151d8 | 747 | #define atomic64_inc(v) atomic64_add(1, (v)) |
1da177e4 LT |
748 | |
749 | /* | |
750 | * atomic64_dec - decrement and test | |
751 | * @v: pointer of type atomic64_t | |
752 | * | |
753 | * Atomically decrements @v by 1. | |
754 | */ | |
21a151d8 | 755 | #define atomic64_dec(v) atomic64_sub(1, (v)) |
1da177e4 LT |
756 | |
757 | /* | |
758 | * atomic64_add_negative - add and test if negative | |
759 | * @v: pointer of type atomic64_t | |
760 | * @i: integer value to add | |
761 | * | |
762 | * Atomically adds @i to @v and returns true | |
763 | * if the result is negative, or false when | |
764 | * result is greater than or equal to zero. | |
765 | */ | |
21a151d8 | 766 | #define atomic64_add_negative(i, v) (atomic64_add_return(i, (v)) < 0) |
1da177e4 | 767 | |
875d43e7 | 768 | #endif /* CONFIG_64BIT */ |
1da177e4 LT |
769 | |
770 | /* | |
771 | * atomic*_return operations are serializing but not the non-*_return | |
772 | * versions. | |
773 | */ | |
f252ffd5 | 774 | #define smp_mb__before_atomic_dec() smp_mb__before_llsc() |
17099b11 | 775 | #define smp_mb__after_atomic_dec() smp_llsc_mb() |
f252ffd5 | 776 | #define smp_mb__before_atomic_inc() smp_mb__before_llsc() |
17099b11 | 777 | #define smp_mb__after_atomic_inc() smp_llsc_mb() |
1da177e4 LT |
778 | |
779 | #endif /* _ASM_ATOMIC_H */ |