Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
feaf7cf1 BB |
2 | #ifndef _ASM_POWERPC_ATOMIC_H_ |
3 | #define _ASM_POWERPC_ATOMIC_H_ | |
4 | ||
1da177e4 LT |
5 | /* |
6 | * PowerPC atomic operations | |
7 | */ | |
8 | ||
1da177e4 | 9 | #ifdef __KERNEL__ |
ae3a197e DH |
10 | #include <linux/types.h> |
11 | #include <asm/cmpxchg.h> | |
c645073f | 12 | #include <asm/barrier.h> |
36a7eeaf | 13 | #include <asm/asm-405.h> |
1da177e4 | 14 | |
feaf7cf1 | 15 | #define ATOMIC_INIT(i) { (i) } |
1da177e4 | 16 | |
dc53617c BF |
17 | /* |
18 | * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with | |
19 | * a "bne-" instruction at the end, so an isync is enough as a acquire barrier | |
20 | * on the platform without lwsync. | |
21 | */ | |
fd2efaa4 MR |
22 | #define __atomic_acquire_fence() \ |
23 | __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory") | |
24 | ||
25 | #define __atomic_release_fence() \ | |
26 | __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory") | |
dc53617c | 27 | |
9f0cbea0 SB |
28 | static __inline__ int atomic_read(const atomic_t *v) |
29 | { | |
30 | int t; | |
31 | ||
32 | __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); | |
33 | ||
34 | return t; | |
35 | } | |
36 | ||
37 | static __inline__ void atomic_set(atomic_t *v, int i) | |
38 | { | |
39 | __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); | |
40 | } | |
1da177e4 | 41 | |
af095dd6 PZ |
42 | #define ATOMIC_OP(op, asm_op) \ |
43 | static __inline__ void atomic_##op(int a, atomic_t *v) \ | |
44 | { \ | |
45 | int t; \ | |
46 | \ | |
47 | __asm__ __volatile__( \ | |
48 | "1: lwarx %0,0,%3 # atomic_" #op "\n" \ | |
49 | #asm_op " %0,%2,%0\n" \ | |
50 | PPC405_ERR77(0,%3) \ | |
51 | " stwcx. %0,0,%3 \n" \ | |
52 | " bne- 1b\n" \ | |
53 | : "=&r" (t), "+m" (v->counter) \ | |
54 | : "r" (a), "r" (&v->counter) \ | |
55 | : "cc"); \ | |
56 | } \ | |
57 | ||
dc53617c BF |
58 | #define ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ |
59 | static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \ | |
af095dd6 PZ |
60 | { \ |
61 | int t; \ | |
62 | \ | |
63 | __asm__ __volatile__( \ | |
dc53617c BF |
64 | "1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \ |
65 | #asm_op " %0,%2,%0\n" \ | |
66 | PPC405_ERR77(0, %3) \ | |
67 | " stwcx. %0,0,%3\n" \ | |
af095dd6 | 68 | " bne- 1b\n" \ |
dc53617c | 69 | : "=&r" (t), "+m" (v->counter) \ |
af095dd6 | 70 | : "r" (a), "r" (&v->counter) \ |
dc53617c | 71 | : "cc"); \ |
af095dd6 PZ |
72 | \ |
73 | return t; \ | |
1da177e4 LT |
74 | } |
75 | ||
a28cc7bb PZ |
76 | #define ATOMIC_FETCH_OP_RELAXED(op, asm_op) \ |
77 | static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ | |
78 | { \ | |
79 | int res, t; \ | |
80 | \ | |
81 | __asm__ __volatile__( \ | |
82 | "1: lwarx %0,0,%4 # atomic_fetch_" #op "_relaxed\n" \ | |
83 | #asm_op " %1,%3,%0\n" \ | |
84 | PPC405_ERR77(0, %4) \ | |
85 | " stwcx. %1,0,%4\n" \ | |
86 | " bne- 1b\n" \ | |
87 | : "=&r" (res), "=&r" (t), "+m" (v->counter) \ | |
88 | : "r" (a), "r" (&v->counter) \ | |
89 | : "cc"); \ | |
90 | \ | |
91 | return res; \ | |
92 | } | |
93 | ||
dc53617c BF |
94 | #define ATOMIC_OPS(op, asm_op) \ |
95 | ATOMIC_OP(op, asm_op) \ | |
a28cc7bb PZ |
96 | ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ |
97 | ATOMIC_FETCH_OP_RELAXED(op, asm_op) | |
1da177e4 | 98 | |
af095dd6 PZ |
99 | ATOMIC_OPS(add, add) |
100 | ATOMIC_OPS(sub, subf) | |
1da177e4 | 101 | |
dc53617c BF |
102 | #define atomic_add_return_relaxed atomic_add_return_relaxed |
103 | #define atomic_sub_return_relaxed atomic_sub_return_relaxed | |
104 | ||
a28cc7bb PZ |
105 | #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed |
106 | #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed | |
107 | ||
108 | #undef ATOMIC_OPS | |
109 | #define ATOMIC_OPS(op, asm_op) \ | |
110 | ATOMIC_OP(op, asm_op) \ | |
111 | ATOMIC_FETCH_OP_RELAXED(op, asm_op) | |
112 | ||
113 | ATOMIC_OPS(and, and) | |
114 | ATOMIC_OPS(or, or) | |
115 | ATOMIC_OPS(xor, xor) | |
116 | ||
117 | #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed | |
118 | #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed | |
119 | #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed | |
120 | ||
af095dd6 | 121 | #undef ATOMIC_OPS |
a28cc7bb | 122 | #undef ATOMIC_FETCH_OP_RELAXED |
dc53617c | 123 | #undef ATOMIC_OP_RETURN_RELAXED |
af095dd6 | 124 | #undef ATOMIC_OP |
1da177e4 | 125 | |
1da177e4 LT |
126 | static __inline__ void atomic_inc(atomic_t *v) |
127 | { | |
128 | int t; | |
129 | ||
130 | __asm__ __volatile__( | |
131 | "1: lwarx %0,0,%2 # atomic_inc\n\ | |
132 | addic %0,%0,1\n" | |
133 | PPC405_ERR77(0,%2) | |
134 | " stwcx. %0,0,%2 \n\ | |
135 | bne- 1b" | |
e2a3d402 LT |
136 | : "=&r" (t), "+m" (v->counter) |
137 | : "r" (&v->counter) | |
efc3624c | 138 | : "cc", "xer"); |
1da177e4 | 139 | } |
9837559d | 140 | #define atomic_inc atomic_inc |
1da177e4 | 141 | |
dc53617c | 142 | static __inline__ int atomic_inc_return_relaxed(atomic_t *v) |
1da177e4 LT |
143 | { |
144 | int t; | |
145 | ||
146 | __asm__ __volatile__( | |
dc53617c BF |
147 | "1: lwarx %0,0,%2 # atomic_inc_return_relaxed\n" |
148 | " addic %0,%0,1\n" | |
149 | PPC405_ERR77(0, %2) | |
150 | " stwcx. %0,0,%2\n" | |
151 | " bne- 1b" | |
152 | : "=&r" (t), "+m" (v->counter) | |
1da177e4 | 153 | : "r" (&v->counter) |
dc53617c | 154 | : "cc", "xer"); |
1da177e4 LT |
155 | |
156 | return t; | |
157 | } | |
158 | ||
1da177e4 LT |
159 | static __inline__ void atomic_dec(atomic_t *v) |
160 | { | |
161 | int t; | |
162 | ||
163 | __asm__ __volatile__( | |
164 | "1: lwarx %0,0,%2 # atomic_dec\n\ | |
165 | addic %0,%0,-1\n" | |
166 | PPC405_ERR77(0,%2)\ | |
167 | " stwcx. %0,0,%2\n\ | |
168 | bne- 1b" | |
e2a3d402 LT |
169 | : "=&r" (t), "+m" (v->counter) |
170 | : "r" (&v->counter) | |
efc3624c | 171 | : "cc", "xer"); |
1da177e4 | 172 | } |
9837559d | 173 | #define atomic_dec atomic_dec |
1da177e4 | 174 | |
dc53617c | 175 | static __inline__ int atomic_dec_return_relaxed(atomic_t *v) |
1da177e4 LT |
176 | { |
177 | int t; | |
178 | ||
179 | __asm__ __volatile__( | |
dc53617c BF |
180 | "1: lwarx %0,0,%2 # atomic_dec_return_relaxed\n" |
181 | " addic %0,%0,-1\n" | |
182 | PPC405_ERR77(0, %2) | |
183 | " stwcx. %0,0,%2\n" | |
184 | " bne- 1b" | |
185 | : "=&r" (t), "+m" (v->counter) | |
1da177e4 | 186 | : "r" (&v->counter) |
dc53617c | 187 | : "cc", "xer"); |
1da177e4 LT |
188 | |
189 | return t; | |
190 | } | |
191 | ||
dc53617c BF |
192 | #define atomic_inc_return_relaxed atomic_inc_return_relaxed |
193 | #define atomic_dec_return_relaxed atomic_dec_return_relaxed | |
194 | ||
f46e477e | 195 | #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) |
56c08e6d BF |
196 | #define atomic_cmpxchg_relaxed(v, o, n) \ |
197 | cmpxchg_relaxed(&((v)->counter), (o), (n)) | |
198 | #define atomic_cmpxchg_acquire(v, o, n) \ | |
199 | cmpxchg_acquire(&((v)->counter), (o), (n)) | |
200 | ||
ffbf670f | 201 | #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) |
26760fc1 | 202 | #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) |
4a6dae6d | 203 | |
8426e1f6 | 204 | /** |
bfc18e38 | 205 | * atomic_fetch_add_unless - add unless the number is a given value |
8426e1f6 NP |
206 | * @v: pointer of type atomic_t |
207 | * @a: the amount to add to v... | |
208 | * @u: ...unless v is equal to u. | |
209 | * | |
210 | * Atomically adds @a to @v, so long as it was not @u. | |
f24219b4 | 211 | * Returns the old value of @v. |
8426e1f6 | 212 | */ |
bfc18e38 | 213 | static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) |
f055affb NP |
214 | { |
215 | int t; | |
216 | ||
217 | __asm__ __volatile__ ( | |
b97021f8 | 218 | PPC_ATOMIC_ENTRY_BARRIER |
bfc18e38 | 219 | "1: lwarx %0,0,%1 # atomic_fetch_add_unless\n\ |
f055affb | 220 | cmpw 0,%0,%3 \n\ |
61e98ebf | 221 | beq 2f \n\ |
f055affb NP |
222 | add %0,%2,%0 \n" |
223 | PPC405_ERR77(0,%2) | |
224 | " stwcx. %0,0,%1 \n\ | |
225 | bne- 1b \n" | |
b97021f8 | 226 | PPC_ATOMIC_EXIT_BARRIER |
f055affb NP |
227 | " subf %0,%2,%0 \n\ |
228 | 2:" | |
229 | : "=&r" (t) | |
230 | : "r" (&v->counter), "r" (a), "r" (u) | |
231 | : "cc", "memory"); | |
232 | ||
f24219b4 | 233 | return t; |
f055affb | 234 | } |
eccc2da8 | 235 | #define atomic_fetch_add_unless atomic_fetch_add_unless |
f055affb | 236 | |
a6cf7ed5 AB |
237 | /** |
238 | * atomic_inc_not_zero - increment unless the number is zero | |
239 | * @v: pointer of type atomic_t | |
240 | * | |
241 | * Atomically increments @v by 1, so long as @v is non-zero. | |
242 | * Returns non-zero if @v was non-zero, and zero otherwise. | |
243 | */ | |
244 | static __inline__ int atomic_inc_not_zero(atomic_t *v) | |
245 | { | |
246 | int t1, t2; | |
247 | ||
248 | __asm__ __volatile__ ( | |
249 | PPC_ATOMIC_ENTRY_BARRIER | |
250 | "1: lwarx %0,0,%2 # atomic_inc_not_zero\n\ | |
251 | cmpwi 0,%0,0\n\ | |
252 | beq- 2f\n\ | |
253 | addic %1,%0,1\n" | |
254 | PPC405_ERR77(0,%2) | |
255 | " stwcx. %1,0,%2\n\ | |
256 | bne- 1b\n" | |
257 | PPC_ATOMIC_EXIT_BARRIER | |
258 | "\n\ | |
259 | 2:" | |
260 | : "=&r" (t1), "=&r" (t2) | |
261 | : "r" (&v->counter) | |
262 | : "cc", "xer", "memory"); | |
263 | ||
264 | return t1; | |
265 | } | |
266 | #define atomic_inc_not_zero(v) atomic_inc_not_zero((v)) | |
8426e1f6 | 267 | |
1da177e4 LT |
268 | /* |
269 | * Atomically test *v and decrement if it is greater than 0. | |
434f98c4 RJ |
270 | * The function returns the old value of *v minus 1, even if |
271 | * the atomic variable, v, was not decremented. | |
1da177e4 LT |
272 | */ |
273 | static __inline__ int atomic_dec_if_positive(atomic_t *v) | |
274 | { | |
275 | int t; | |
276 | ||
277 | __asm__ __volatile__( | |
b97021f8 | 278 | PPC_ATOMIC_ENTRY_BARRIER |
1da177e4 | 279 | "1: lwarx %0,0,%1 # atomic_dec_if_positive\n\ |
434f98c4 RJ |
280 | cmpwi %0,1\n\ |
281 | addi %0,%0,-1\n\ | |
1da177e4 LT |
282 | blt- 2f\n" |
283 | PPC405_ERR77(0,%1) | |
284 | " stwcx. %0,0,%1\n\ | |
285 | bne- 1b" | |
b97021f8 | 286 | PPC_ATOMIC_EXIT_BARRIER |
1da177e4 | 287 | "\n\ |
434f98c4 | 288 | 2:" : "=&b" (t) |
1da177e4 LT |
289 | : "r" (&v->counter) |
290 | : "cc", "memory"); | |
291 | ||
292 | return t; | |
293 | } | |
e79bee24 | 294 | #define atomic_dec_if_positive atomic_dec_if_positive |
1da177e4 | 295 | |
06a98dba SR |
296 | #ifdef __powerpc64__ |
297 | ||
06a98dba SR |
298 | #define ATOMIC64_INIT(i) { (i) } |
299 | ||
9f0cbea0 SB |
300 | static __inline__ long atomic64_read(const atomic64_t *v) |
301 | { | |
302 | long t; | |
303 | ||
304 | __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); | |
305 | ||
306 | return t; | |
307 | } | |
308 | ||
309 | static __inline__ void atomic64_set(atomic64_t *v, long i) | |
310 | { | |
311 | __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); | |
312 | } | |
06a98dba | 313 | |
af095dd6 PZ |
314 | #define ATOMIC64_OP(op, asm_op) \ |
315 | static __inline__ void atomic64_##op(long a, atomic64_t *v) \ | |
316 | { \ | |
317 | long t; \ | |
318 | \ | |
319 | __asm__ __volatile__( \ | |
320 | "1: ldarx %0,0,%3 # atomic64_" #op "\n" \ | |
321 | #asm_op " %0,%2,%0\n" \ | |
322 | " stdcx. %0,0,%3 \n" \ | |
323 | " bne- 1b\n" \ | |
324 | : "=&r" (t), "+m" (v->counter) \ | |
325 | : "r" (a), "r" (&v->counter) \ | |
326 | : "cc"); \ | |
06a98dba SR |
327 | } |
328 | ||
dc53617c BF |
329 | #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ |
330 | static inline long \ | |
331 | atomic64_##op##_return_relaxed(long a, atomic64_t *v) \ | |
af095dd6 PZ |
332 | { \ |
333 | long t; \ | |
334 | \ | |
335 | __asm__ __volatile__( \ | |
dc53617c BF |
336 | "1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \ |
337 | #asm_op " %0,%2,%0\n" \ | |
338 | " stdcx. %0,0,%3\n" \ | |
af095dd6 | 339 | " bne- 1b\n" \ |
dc53617c | 340 | : "=&r" (t), "+m" (v->counter) \ |
af095dd6 | 341 | : "r" (a), "r" (&v->counter) \ |
dc53617c | 342 | : "cc"); \ |
af095dd6 PZ |
343 | \ |
344 | return t; \ | |
06a98dba SR |
345 | } |
346 | ||
a28cc7bb PZ |
347 | #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \ |
348 | static inline long \ | |
349 | atomic64_fetch_##op##_relaxed(long a, atomic64_t *v) \ | |
350 | { \ | |
351 | long res, t; \ | |
352 | \ | |
353 | __asm__ __volatile__( \ | |
354 | "1: ldarx %0,0,%4 # atomic64_fetch_" #op "_relaxed\n" \ | |
355 | #asm_op " %1,%3,%0\n" \ | |
356 | " stdcx. %1,0,%4\n" \ | |
357 | " bne- 1b\n" \ | |
358 | : "=&r" (res), "=&r" (t), "+m" (v->counter) \ | |
359 | : "r" (a), "r" (&v->counter) \ | |
360 | : "cc"); \ | |
361 | \ | |
362 | return res; \ | |
363 | } | |
364 | ||
dc53617c BF |
365 | #define ATOMIC64_OPS(op, asm_op) \ |
366 | ATOMIC64_OP(op, asm_op) \ | |
a28cc7bb PZ |
367 | ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ |
368 | ATOMIC64_FETCH_OP_RELAXED(op, asm_op) | |
06a98dba | 369 | |
af095dd6 PZ |
370 | ATOMIC64_OPS(add, add) |
371 | ATOMIC64_OPS(sub, subf) | |
06a98dba | 372 | |
dc53617c BF |
373 | #define atomic64_add_return_relaxed atomic64_add_return_relaxed |
374 | #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed | |
375 | ||
a28cc7bb PZ |
376 | #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed |
377 | #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed | |
378 | ||
379 | #undef ATOMIC64_OPS | |
380 | #define ATOMIC64_OPS(op, asm_op) \ | |
381 | ATOMIC64_OP(op, asm_op) \ | |
382 | ATOMIC64_FETCH_OP_RELAXED(op, asm_op) | |
383 | ||
384 | ATOMIC64_OPS(and, and) | |
385 | ATOMIC64_OPS(or, or) | |
386 | ATOMIC64_OPS(xor, xor) | |
387 | ||
388 | #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed | |
389 | #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed | |
390 | #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed | |
391 | ||
dc53617c | 392 | #undef ATOPIC64_OPS |
a28cc7bb | 393 | #undef ATOMIC64_FETCH_OP_RELAXED |
dc53617c | 394 | #undef ATOMIC64_OP_RETURN_RELAXED |
af095dd6 | 395 | #undef ATOMIC64_OP |
06a98dba | 396 | |
06a98dba SR |
397 | static __inline__ void atomic64_inc(atomic64_t *v) |
398 | { | |
399 | long t; | |
400 | ||
401 | __asm__ __volatile__( | |
402 | "1: ldarx %0,0,%2 # atomic64_inc\n\ | |
403 | addic %0,%0,1\n\ | |
404 | stdcx. %0,0,%2 \n\ | |
405 | bne- 1b" | |
e2a3d402 LT |
406 | : "=&r" (t), "+m" (v->counter) |
407 | : "r" (&v->counter) | |
efc3624c | 408 | : "cc", "xer"); |
06a98dba | 409 | } |
9837559d | 410 | #define atomic64_inc atomic64_inc |
06a98dba | 411 | |
dc53617c | 412 | static __inline__ long atomic64_inc_return_relaxed(atomic64_t *v) |
06a98dba SR |
413 | { |
414 | long t; | |
415 | ||
416 | __asm__ __volatile__( | |
dc53617c BF |
417 | "1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n" |
418 | " addic %0,%0,1\n" | |
419 | " stdcx. %0,0,%2\n" | |
420 | " bne- 1b" | |
421 | : "=&r" (t), "+m" (v->counter) | |
06a98dba | 422 | : "r" (&v->counter) |
dc53617c | 423 | : "cc", "xer"); |
06a98dba SR |
424 | |
425 | return t; | |
426 | } | |
427 | ||
06a98dba SR |
428 | static __inline__ void atomic64_dec(atomic64_t *v) |
429 | { | |
430 | long t; | |
431 | ||
432 | __asm__ __volatile__( | |
433 | "1: ldarx %0,0,%2 # atomic64_dec\n\ | |
434 | addic %0,%0,-1\n\ | |
435 | stdcx. %0,0,%2\n\ | |
436 | bne- 1b" | |
e2a3d402 LT |
437 | : "=&r" (t), "+m" (v->counter) |
438 | : "r" (&v->counter) | |
efc3624c | 439 | : "cc", "xer"); |
06a98dba | 440 | } |
9837559d | 441 | #define atomic64_dec atomic64_dec |
06a98dba | 442 | |
dc53617c | 443 | static __inline__ long atomic64_dec_return_relaxed(atomic64_t *v) |
06a98dba SR |
444 | { |
445 | long t; | |
446 | ||
447 | __asm__ __volatile__( | |
dc53617c BF |
448 | "1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n" |
449 | " addic %0,%0,-1\n" | |
450 | " stdcx. %0,0,%2\n" | |
451 | " bne- 1b" | |
452 | : "=&r" (t), "+m" (v->counter) | |
06a98dba | 453 | : "r" (&v->counter) |
dc53617c | 454 | : "cc", "xer"); |
06a98dba SR |
455 | |
456 | return t; | |
457 | } | |
458 | ||
dc53617c BF |
459 | #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed |
460 | #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed | |
461 | ||
06a98dba SR |
462 | /* |
463 | * Atomically test *v and decrement if it is greater than 0. | |
464 | * The function returns the old value of *v minus 1. | |
465 | */ | |
466 | static __inline__ long atomic64_dec_if_positive(atomic64_t *v) | |
467 | { | |
468 | long t; | |
469 | ||
470 | __asm__ __volatile__( | |
b97021f8 | 471 | PPC_ATOMIC_ENTRY_BARRIER |
06a98dba SR |
472 | "1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\ |
473 | addic. %0,%0,-1\n\ | |
474 | blt- 2f\n\ | |
475 | stdcx. %0,0,%1\n\ | |
476 | bne- 1b" | |
b97021f8 | 477 | PPC_ATOMIC_EXIT_BARRIER |
06a98dba SR |
478 | "\n\ |
479 | 2:" : "=&r" (t) | |
480 | : "r" (&v->counter) | |
efc3624c | 481 | : "cc", "xer", "memory"); |
06a98dba SR |
482 | |
483 | return t; | |
484 | } | |
b3a2a05f | 485 | #define atomic64_dec_if_positive atomic64_dec_if_positive |
06a98dba | 486 | |
f46e477e | 487 | #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) |
56c08e6d BF |
488 | #define atomic64_cmpxchg_relaxed(v, o, n) \ |
489 | cmpxchg_relaxed(&((v)->counter), (o), (n)) | |
490 | #define atomic64_cmpxchg_acquire(v, o, n) \ | |
491 | cmpxchg_acquire(&((v)->counter), (o), (n)) | |
492 | ||
41806ef4 | 493 | #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) |
26760fc1 | 494 | #define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) |
41806ef4 MD |
495 | |
496 | /** | |
4f44b4b2 | 497 | * atomic64_fetch_add_unless - add unless the number is a given value |
41806ef4 MD |
498 | * @v: pointer of type atomic64_t |
499 | * @a: the amount to add to v... | |
500 | * @u: ...unless v is equal to u. | |
501 | * | |
502 | * Atomically adds @a to @v, so long as it was not @u. | |
f24219b4 | 503 | * Returns the old value of @v. |
41806ef4 | 504 | */ |
4f44b4b2 | 505 | static __inline__ long atomic64_fetch_add_unless(atomic64_t *v, long a, long u) |
41806ef4 MD |
506 | { |
507 | long t; | |
508 | ||
509 | __asm__ __volatile__ ( | |
b97021f8 | 510 | PPC_ATOMIC_ENTRY_BARRIER |
4f44b4b2 | 511 | "1: ldarx %0,0,%1 # atomic64_fetch_add_unless\n\ |
41806ef4 | 512 | cmpd 0,%0,%3 \n\ |
61e98ebf | 513 | beq 2f \n\ |
41806ef4 MD |
514 | add %0,%2,%0 \n" |
515 | " stdcx. %0,0,%1 \n\ | |
516 | bne- 1b \n" | |
b97021f8 | 517 | PPC_ATOMIC_EXIT_BARRIER |
41806ef4 MD |
518 | " subf %0,%2,%0 \n\ |
519 | 2:" | |
520 | : "=&r" (t) | |
521 | : "r" (&v->counter), "r" (a), "r" (u) | |
522 | : "cc", "memory"); | |
523 | ||
4f44b4b2 | 524 | return t; |
41806ef4 | 525 | } |
4f44b4b2 | 526 | #define atomic64_fetch_add_unless atomic64_fetch_add_unless |
41806ef4 | 527 | |
a6cf7ed5 AB |
528 | /** |
529 | * atomic_inc64_not_zero - increment unless the number is zero | |
530 | * @v: pointer of type atomic64_t | |
531 | * | |
532 | * Atomically increments @v by 1, so long as @v is non-zero. | |
533 | * Returns non-zero if @v was non-zero, and zero otherwise. | |
534 | */ | |
01e6a61a | 535 | static __inline__ int atomic64_inc_not_zero(atomic64_t *v) |
a6cf7ed5 AB |
536 | { |
537 | long t1, t2; | |
538 | ||
539 | __asm__ __volatile__ ( | |
540 | PPC_ATOMIC_ENTRY_BARRIER | |
541 | "1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\ | |
542 | cmpdi 0,%0,0\n\ | |
543 | beq- 2f\n\ | |
544 | addic %1,%0,1\n\ | |
545 | stdcx. %1,0,%2\n\ | |
546 | bne- 1b\n" | |
547 | PPC_ATOMIC_EXIT_BARRIER | |
548 | "\n\ | |
549 | 2:" | |
550 | : "=&r" (t1), "=&r" (t2) | |
551 | : "r" (&v->counter) | |
552 | : "cc", "xer", "memory"); | |
553 | ||
01e6a61a | 554 | return t1 != 0; |
a6cf7ed5 | 555 | } |
bef82820 | 556 | #define atomic64_inc_not_zero(v) atomic64_inc_not_zero((v)) |
41806ef4 | 557 | |
06a98dba SR |
558 | #endif /* __powerpc64__ */ |
559 | ||
1da177e4 | 560 | #endif /* __KERNEL__ */ |
feaf7cf1 | 561 | #endif /* _ASM_POWERPC_ATOMIC_H_ */ |