Commit | Line | Data |
---|---|---|
aa525d06 MR |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | // Generated by scripts/atomic/gen-atomic-instrumented.sh | |
4 | // DO NOT MODIFY THIS FILE DIRECTLY | |
5 | ||
ac605bee DV |
6 | /* |
7 | * This file provides wrappers with KASAN instrumentation for atomic operations. | |
8 | * To use this functionality an arch's atomic.h file needs to define all | |
9 | * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include | |
10 | * this file at the end. This file provides atomic_read() that forwards to | |
11 | * arch_atomic_read() for actual atomic operation. | |
12 | * Note: if an arch atomic operation is implemented by means of other atomic | |
13 | * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use | |
14 | * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid | |
15 | * double instrumentation. | |
16 | */ | |
aa525d06 MR |
17 | #ifndef _ASM_GENERIC_ATOMIC_INSTRUMENTED_H |
18 | #define _ASM_GENERIC_ATOMIC_INSTRUMENTED_H | |
19 | ||
20 | #include <linux/build_bug.h> | |
c020395b | 21 | #include <linux/compiler.h> |
ed8af2e4 | 22 | #include <linux/instrumented.h> |
aa525d06 | 23 | |
c020395b | 24 | static __always_inline int |
aa525d06 MR |
25 | atomic_read(const atomic_t *v) |
26 | { | |
ed8af2e4 | 27 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
28 | return arch_atomic_read(v); |
29 | } | |
30 | #define atomic_read atomic_read | |
31 | ||
32 | #if defined(arch_atomic_read_acquire) | |
c020395b | 33 | static __always_inline int |
aa525d06 MR |
34 | atomic_read_acquire(const atomic_t *v) |
35 | { | |
ed8af2e4 | 36 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
37 | return arch_atomic_read_acquire(v); |
38 | } | |
39 | #define atomic_read_acquire atomic_read_acquire | |
40 | #endif | |
41 | ||
c020395b | 42 | static __always_inline void |
aa525d06 MR |
43 | atomic_set(atomic_t *v, int i) |
44 | { | |
ed8af2e4 | 45 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
46 | arch_atomic_set(v, i); |
47 | } | |
48 | #define atomic_set atomic_set | |
49 | ||
50 | #if defined(arch_atomic_set_release) | |
c020395b | 51 | static __always_inline void |
aa525d06 MR |
52 | atomic_set_release(atomic_t *v, int i) |
53 | { | |
ed8af2e4 | 54 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
55 | arch_atomic_set_release(v, i); |
56 | } | |
57 | #define atomic_set_release atomic_set_release | |
58 | #endif | |
59 | ||
c020395b | 60 | static __always_inline void |
aa525d06 MR |
61 | atomic_add(int i, atomic_t *v) |
62 | { | |
ed8af2e4 | 63 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
64 | arch_atomic_add(i, v); |
65 | } | |
66 | #define atomic_add atomic_add | |
67 | ||
68 | #if !defined(arch_atomic_add_return_relaxed) || defined(arch_atomic_add_return) | |
c020395b | 69 | static __always_inline int |
aa525d06 MR |
70 | atomic_add_return(int i, atomic_t *v) |
71 | { | |
ed8af2e4 | 72 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
73 | return arch_atomic_add_return(i, v); |
74 | } | |
75 | #define atomic_add_return atomic_add_return | |
76 | #endif | |
77 | ||
78 | #if defined(arch_atomic_add_return_acquire) | |
c020395b | 79 | static __always_inline int |
aa525d06 MR |
80 | atomic_add_return_acquire(int i, atomic_t *v) |
81 | { | |
ed8af2e4 | 82 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
83 | return arch_atomic_add_return_acquire(i, v); |
84 | } | |
85 | #define atomic_add_return_acquire atomic_add_return_acquire | |
86 | #endif | |
87 | ||
88 | #if defined(arch_atomic_add_return_release) | |
c020395b | 89 | static __always_inline int |
aa525d06 MR |
90 | atomic_add_return_release(int i, atomic_t *v) |
91 | { | |
ed8af2e4 | 92 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
93 | return arch_atomic_add_return_release(i, v); |
94 | } | |
95 | #define atomic_add_return_release atomic_add_return_release | |
96 | #endif | |
97 | ||
98 | #if defined(arch_atomic_add_return_relaxed) | |
c020395b | 99 | static __always_inline int |
aa525d06 MR |
100 | atomic_add_return_relaxed(int i, atomic_t *v) |
101 | { | |
ed8af2e4 | 102 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
103 | return arch_atomic_add_return_relaxed(i, v); |
104 | } | |
105 | #define atomic_add_return_relaxed atomic_add_return_relaxed | |
106 | #endif | |
107 | ||
108 | #if !defined(arch_atomic_fetch_add_relaxed) || defined(arch_atomic_fetch_add) | |
c020395b | 109 | static __always_inline int |
aa525d06 MR |
110 | atomic_fetch_add(int i, atomic_t *v) |
111 | { | |
ed8af2e4 | 112 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
113 | return arch_atomic_fetch_add(i, v); |
114 | } | |
115 | #define atomic_fetch_add atomic_fetch_add | |
116 | #endif | |
117 | ||
118 | #if defined(arch_atomic_fetch_add_acquire) | |
c020395b | 119 | static __always_inline int |
aa525d06 MR |
120 | atomic_fetch_add_acquire(int i, atomic_t *v) |
121 | { | |
ed8af2e4 | 122 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
123 | return arch_atomic_fetch_add_acquire(i, v); |
124 | } | |
125 | #define atomic_fetch_add_acquire atomic_fetch_add_acquire | |
126 | #endif | |
127 | ||
128 | #if defined(arch_atomic_fetch_add_release) | |
c020395b | 129 | static __always_inline int |
aa525d06 MR |
130 | atomic_fetch_add_release(int i, atomic_t *v) |
131 | { | |
ed8af2e4 | 132 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
133 | return arch_atomic_fetch_add_release(i, v); |
134 | } | |
135 | #define atomic_fetch_add_release atomic_fetch_add_release | |
136 | #endif | |
137 | ||
138 | #if defined(arch_atomic_fetch_add_relaxed) | |
c020395b | 139 | static __always_inline int |
aa525d06 MR |
140 | atomic_fetch_add_relaxed(int i, atomic_t *v) |
141 | { | |
ed8af2e4 | 142 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
143 | return arch_atomic_fetch_add_relaxed(i, v); |
144 | } | |
145 | #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed | |
146 | #endif | |
147 | ||
c020395b | 148 | static __always_inline void |
aa525d06 MR |
149 | atomic_sub(int i, atomic_t *v) |
150 | { | |
ed8af2e4 | 151 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
152 | arch_atomic_sub(i, v); |
153 | } | |
154 | #define atomic_sub atomic_sub | |
155 | ||
156 | #if !defined(arch_atomic_sub_return_relaxed) || defined(arch_atomic_sub_return) | |
c020395b | 157 | static __always_inline int |
aa525d06 MR |
158 | atomic_sub_return(int i, atomic_t *v) |
159 | { | |
ed8af2e4 | 160 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
161 | return arch_atomic_sub_return(i, v); |
162 | } | |
163 | #define atomic_sub_return atomic_sub_return | |
164 | #endif | |
165 | ||
166 | #if defined(arch_atomic_sub_return_acquire) | |
c020395b | 167 | static __always_inline int |
aa525d06 MR |
168 | atomic_sub_return_acquire(int i, atomic_t *v) |
169 | { | |
ed8af2e4 | 170 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
171 | return arch_atomic_sub_return_acquire(i, v); |
172 | } | |
173 | #define atomic_sub_return_acquire atomic_sub_return_acquire | |
174 | #endif | |
175 | ||
176 | #if defined(arch_atomic_sub_return_release) | |
c020395b | 177 | static __always_inline int |
aa525d06 MR |
178 | atomic_sub_return_release(int i, atomic_t *v) |
179 | { | |
ed8af2e4 | 180 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
181 | return arch_atomic_sub_return_release(i, v); |
182 | } | |
183 | #define atomic_sub_return_release atomic_sub_return_release | |
184 | #endif | |
185 | ||
186 | #if defined(arch_atomic_sub_return_relaxed) | |
c020395b | 187 | static __always_inline int |
aa525d06 MR |
188 | atomic_sub_return_relaxed(int i, atomic_t *v) |
189 | { | |
ed8af2e4 | 190 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
191 | return arch_atomic_sub_return_relaxed(i, v); |
192 | } | |
193 | #define atomic_sub_return_relaxed atomic_sub_return_relaxed | |
194 | #endif | |
195 | ||
196 | #if !defined(arch_atomic_fetch_sub_relaxed) || defined(arch_atomic_fetch_sub) | |
c020395b | 197 | static __always_inline int |
aa525d06 MR |
198 | atomic_fetch_sub(int i, atomic_t *v) |
199 | { | |
ed8af2e4 | 200 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
201 | return arch_atomic_fetch_sub(i, v); |
202 | } | |
203 | #define atomic_fetch_sub atomic_fetch_sub | |
204 | #endif | |
205 | ||
206 | #if defined(arch_atomic_fetch_sub_acquire) | |
c020395b | 207 | static __always_inline int |
aa525d06 MR |
208 | atomic_fetch_sub_acquire(int i, atomic_t *v) |
209 | { | |
ed8af2e4 | 210 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
211 | return arch_atomic_fetch_sub_acquire(i, v); |
212 | } | |
213 | #define atomic_fetch_sub_acquire atomic_fetch_sub_acquire | |
214 | #endif | |
215 | ||
216 | #if defined(arch_atomic_fetch_sub_release) | |
c020395b | 217 | static __always_inline int |
aa525d06 MR |
218 | atomic_fetch_sub_release(int i, atomic_t *v) |
219 | { | |
ed8af2e4 | 220 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
221 | return arch_atomic_fetch_sub_release(i, v); |
222 | } | |
223 | #define atomic_fetch_sub_release atomic_fetch_sub_release | |
224 | #endif | |
225 | ||
226 | #if defined(arch_atomic_fetch_sub_relaxed) | |
c020395b | 227 | static __always_inline int |
aa525d06 MR |
228 | atomic_fetch_sub_relaxed(int i, atomic_t *v) |
229 | { | |
ed8af2e4 | 230 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
231 | return arch_atomic_fetch_sub_relaxed(i, v); |
232 | } | |
233 | #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed | |
234 | #endif | |
235 | ||
236 | #if defined(arch_atomic_inc) | |
c020395b | 237 | static __always_inline void |
aa525d06 MR |
238 | atomic_inc(atomic_t *v) |
239 | { | |
ed8af2e4 | 240 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
241 | arch_atomic_inc(v); |
242 | } | |
243 | #define atomic_inc atomic_inc | |
244 | #endif | |
245 | ||
246 | #if defined(arch_atomic_inc_return) | |
c020395b | 247 | static __always_inline int |
aa525d06 MR |
248 | atomic_inc_return(atomic_t *v) |
249 | { | |
ed8af2e4 | 250 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
251 | return arch_atomic_inc_return(v); |
252 | } | |
253 | #define atomic_inc_return atomic_inc_return | |
254 | #endif | |
255 | ||
256 | #if defined(arch_atomic_inc_return_acquire) | |
c020395b | 257 | static __always_inline int |
aa525d06 MR |
258 | atomic_inc_return_acquire(atomic_t *v) |
259 | { | |
ed8af2e4 | 260 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
261 | return arch_atomic_inc_return_acquire(v); |
262 | } | |
263 | #define atomic_inc_return_acquire atomic_inc_return_acquire | |
264 | #endif | |
265 | ||
266 | #if defined(arch_atomic_inc_return_release) | |
c020395b | 267 | static __always_inline int |
aa525d06 MR |
268 | atomic_inc_return_release(atomic_t *v) |
269 | { | |
ed8af2e4 | 270 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
271 | return arch_atomic_inc_return_release(v); |
272 | } | |
273 | #define atomic_inc_return_release atomic_inc_return_release | |
274 | #endif | |
275 | ||
276 | #if defined(arch_atomic_inc_return_relaxed) | |
c020395b | 277 | static __always_inline int |
aa525d06 MR |
278 | atomic_inc_return_relaxed(atomic_t *v) |
279 | { | |
ed8af2e4 | 280 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
281 | return arch_atomic_inc_return_relaxed(v); |
282 | } | |
283 | #define atomic_inc_return_relaxed atomic_inc_return_relaxed | |
284 | #endif | |
285 | ||
286 | #if defined(arch_atomic_fetch_inc) | |
c020395b | 287 | static __always_inline int |
aa525d06 MR |
288 | atomic_fetch_inc(atomic_t *v) |
289 | { | |
ed8af2e4 | 290 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
291 | return arch_atomic_fetch_inc(v); |
292 | } | |
293 | #define atomic_fetch_inc atomic_fetch_inc | |
294 | #endif | |
295 | ||
296 | #if defined(arch_atomic_fetch_inc_acquire) | |
c020395b | 297 | static __always_inline int |
aa525d06 MR |
298 | atomic_fetch_inc_acquire(atomic_t *v) |
299 | { | |
ed8af2e4 | 300 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
301 | return arch_atomic_fetch_inc_acquire(v); |
302 | } | |
303 | #define atomic_fetch_inc_acquire atomic_fetch_inc_acquire | |
304 | #endif | |
305 | ||
306 | #if defined(arch_atomic_fetch_inc_release) | |
c020395b | 307 | static __always_inline int |
aa525d06 MR |
308 | atomic_fetch_inc_release(atomic_t *v) |
309 | { | |
ed8af2e4 | 310 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
311 | return arch_atomic_fetch_inc_release(v); |
312 | } | |
313 | #define atomic_fetch_inc_release atomic_fetch_inc_release | |
314 | #endif | |
315 | ||
316 | #if defined(arch_atomic_fetch_inc_relaxed) | |
c020395b | 317 | static __always_inline int |
aa525d06 MR |
318 | atomic_fetch_inc_relaxed(atomic_t *v) |
319 | { | |
ed8af2e4 | 320 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
321 | return arch_atomic_fetch_inc_relaxed(v); |
322 | } | |
323 | #define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed | |
324 | #endif | |
325 | ||
326 | #if defined(arch_atomic_dec) | |
c020395b | 327 | static __always_inline void |
aa525d06 MR |
328 | atomic_dec(atomic_t *v) |
329 | { | |
ed8af2e4 | 330 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
331 | arch_atomic_dec(v); |
332 | } | |
333 | #define atomic_dec atomic_dec | |
334 | #endif | |
335 | ||
336 | #if defined(arch_atomic_dec_return) | |
c020395b | 337 | static __always_inline int |
aa525d06 MR |
338 | atomic_dec_return(atomic_t *v) |
339 | { | |
ed8af2e4 | 340 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
341 | return arch_atomic_dec_return(v); |
342 | } | |
343 | #define atomic_dec_return atomic_dec_return | |
344 | #endif | |
345 | ||
346 | #if defined(arch_atomic_dec_return_acquire) | |
c020395b | 347 | static __always_inline int |
aa525d06 MR |
348 | atomic_dec_return_acquire(atomic_t *v) |
349 | { | |
ed8af2e4 | 350 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
351 | return arch_atomic_dec_return_acquire(v); |
352 | } | |
353 | #define atomic_dec_return_acquire atomic_dec_return_acquire | |
354 | #endif | |
355 | ||
356 | #if defined(arch_atomic_dec_return_release) | |
c020395b | 357 | static __always_inline int |
aa525d06 MR |
358 | atomic_dec_return_release(atomic_t *v) |
359 | { | |
ed8af2e4 | 360 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
361 | return arch_atomic_dec_return_release(v); |
362 | } | |
363 | #define atomic_dec_return_release atomic_dec_return_release | |
364 | #endif | |
365 | ||
366 | #if defined(arch_atomic_dec_return_relaxed) | |
c020395b | 367 | static __always_inline int |
aa525d06 MR |
368 | atomic_dec_return_relaxed(atomic_t *v) |
369 | { | |
ed8af2e4 | 370 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
371 | return arch_atomic_dec_return_relaxed(v); |
372 | } | |
373 | #define atomic_dec_return_relaxed atomic_dec_return_relaxed | |
374 | #endif | |
375 | ||
376 | #if defined(arch_atomic_fetch_dec) | |
c020395b | 377 | static __always_inline int |
aa525d06 MR |
378 | atomic_fetch_dec(atomic_t *v) |
379 | { | |
ed8af2e4 | 380 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
381 | return arch_atomic_fetch_dec(v); |
382 | } | |
383 | #define atomic_fetch_dec atomic_fetch_dec | |
384 | #endif | |
385 | ||
386 | #if defined(arch_atomic_fetch_dec_acquire) | |
c020395b | 387 | static __always_inline int |
aa525d06 MR |
388 | atomic_fetch_dec_acquire(atomic_t *v) |
389 | { | |
ed8af2e4 | 390 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
391 | return arch_atomic_fetch_dec_acquire(v); |
392 | } | |
393 | #define atomic_fetch_dec_acquire atomic_fetch_dec_acquire | |
394 | #endif | |
395 | ||
396 | #if defined(arch_atomic_fetch_dec_release) | |
c020395b | 397 | static __always_inline int |
aa525d06 MR |
398 | atomic_fetch_dec_release(atomic_t *v) |
399 | { | |
ed8af2e4 | 400 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
401 | return arch_atomic_fetch_dec_release(v); |
402 | } | |
403 | #define atomic_fetch_dec_release atomic_fetch_dec_release | |
404 | #endif | |
405 | ||
406 | #if defined(arch_atomic_fetch_dec_relaxed) | |
c020395b | 407 | static __always_inline int |
aa525d06 MR |
408 | atomic_fetch_dec_relaxed(atomic_t *v) |
409 | { | |
ed8af2e4 | 410 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
411 | return arch_atomic_fetch_dec_relaxed(v); |
412 | } | |
413 | #define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed | |
414 | #endif | |
415 | ||
c020395b | 416 | static __always_inline void |
aa525d06 MR |
417 | atomic_and(int i, atomic_t *v) |
418 | { | |
ed8af2e4 | 419 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
420 | arch_atomic_and(i, v); |
421 | } | |
422 | #define atomic_and atomic_and | |
423 | ||
424 | #if !defined(arch_atomic_fetch_and_relaxed) || defined(arch_atomic_fetch_and) | |
c020395b | 425 | static __always_inline int |
aa525d06 MR |
426 | atomic_fetch_and(int i, atomic_t *v) |
427 | { | |
ed8af2e4 | 428 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
429 | return arch_atomic_fetch_and(i, v); |
430 | } | |
431 | #define atomic_fetch_and atomic_fetch_and | |
432 | #endif | |
433 | ||
434 | #if defined(arch_atomic_fetch_and_acquire) | |
c020395b | 435 | static __always_inline int |
aa525d06 MR |
436 | atomic_fetch_and_acquire(int i, atomic_t *v) |
437 | { | |
ed8af2e4 | 438 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
439 | return arch_atomic_fetch_and_acquire(i, v); |
440 | } | |
441 | #define atomic_fetch_and_acquire atomic_fetch_and_acquire | |
442 | #endif | |
443 | ||
444 | #if defined(arch_atomic_fetch_and_release) | |
c020395b | 445 | static __always_inline int |
aa525d06 MR |
446 | atomic_fetch_and_release(int i, atomic_t *v) |
447 | { | |
ed8af2e4 | 448 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
449 | return arch_atomic_fetch_and_release(i, v); |
450 | } | |
451 | #define atomic_fetch_and_release atomic_fetch_and_release | |
452 | #endif | |
453 | ||
454 | #if defined(arch_atomic_fetch_and_relaxed) | |
c020395b | 455 | static __always_inline int |
aa525d06 MR |
456 | atomic_fetch_and_relaxed(int i, atomic_t *v) |
457 | { | |
ed8af2e4 | 458 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
459 | return arch_atomic_fetch_and_relaxed(i, v); |
460 | } | |
461 | #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed | |
462 | #endif | |
463 | ||
464 | #if defined(arch_atomic_andnot) | |
c020395b | 465 | static __always_inline void |
aa525d06 MR |
466 | atomic_andnot(int i, atomic_t *v) |
467 | { | |
ed8af2e4 | 468 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
469 | arch_atomic_andnot(i, v); |
470 | } | |
471 | #define atomic_andnot atomic_andnot | |
472 | #endif | |
473 | ||
474 | #if defined(arch_atomic_fetch_andnot) | |
c020395b | 475 | static __always_inline int |
aa525d06 MR |
476 | atomic_fetch_andnot(int i, atomic_t *v) |
477 | { | |
ed8af2e4 | 478 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
479 | return arch_atomic_fetch_andnot(i, v); |
480 | } | |
481 | #define atomic_fetch_andnot atomic_fetch_andnot | |
482 | #endif | |
483 | ||
484 | #if defined(arch_atomic_fetch_andnot_acquire) | |
c020395b | 485 | static __always_inline int |
aa525d06 MR |
486 | atomic_fetch_andnot_acquire(int i, atomic_t *v) |
487 | { | |
ed8af2e4 | 488 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
489 | return arch_atomic_fetch_andnot_acquire(i, v); |
490 | } | |
491 | #define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire | |
492 | #endif | |
493 | ||
494 | #if defined(arch_atomic_fetch_andnot_release) | |
c020395b | 495 | static __always_inline int |
aa525d06 MR |
496 | atomic_fetch_andnot_release(int i, atomic_t *v) |
497 | { | |
ed8af2e4 | 498 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
499 | return arch_atomic_fetch_andnot_release(i, v); |
500 | } | |
501 | #define atomic_fetch_andnot_release atomic_fetch_andnot_release | |
502 | #endif | |
503 | ||
504 | #if defined(arch_atomic_fetch_andnot_relaxed) | |
c020395b | 505 | static __always_inline int |
aa525d06 MR |
506 | atomic_fetch_andnot_relaxed(int i, atomic_t *v) |
507 | { | |
ed8af2e4 | 508 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
509 | return arch_atomic_fetch_andnot_relaxed(i, v); |
510 | } | |
511 | #define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed | |
512 | #endif | |
513 | ||
c020395b | 514 | static __always_inline void |
aa525d06 MR |
515 | atomic_or(int i, atomic_t *v) |
516 | { | |
ed8af2e4 | 517 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
518 | arch_atomic_or(i, v); |
519 | } | |
520 | #define atomic_or atomic_or | |
521 | ||
522 | #if !defined(arch_atomic_fetch_or_relaxed) || defined(arch_atomic_fetch_or) | |
c020395b | 523 | static __always_inline int |
aa525d06 MR |
524 | atomic_fetch_or(int i, atomic_t *v) |
525 | { | |
ed8af2e4 | 526 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
527 | return arch_atomic_fetch_or(i, v); |
528 | } | |
529 | #define atomic_fetch_or atomic_fetch_or | |
530 | #endif | |
531 | ||
532 | #if defined(arch_atomic_fetch_or_acquire) | |
c020395b | 533 | static __always_inline int |
aa525d06 MR |
534 | atomic_fetch_or_acquire(int i, atomic_t *v) |
535 | { | |
ed8af2e4 | 536 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
537 | return arch_atomic_fetch_or_acquire(i, v); |
538 | } | |
539 | #define atomic_fetch_or_acquire atomic_fetch_or_acquire | |
540 | #endif | |
541 | ||
542 | #if defined(arch_atomic_fetch_or_release) | |
c020395b | 543 | static __always_inline int |
aa525d06 MR |
544 | atomic_fetch_or_release(int i, atomic_t *v) |
545 | { | |
ed8af2e4 | 546 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
547 | return arch_atomic_fetch_or_release(i, v); |
548 | } | |
549 | #define atomic_fetch_or_release atomic_fetch_or_release | |
550 | #endif | |
551 | ||
552 | #if defined(arch_atomic_fetch_or_relaxed) | |
c020395b | 553 | static __always_inline int |
aa525d06 MR |
554 | atomic_fetch_or_relaxed(int i, atomic_t *v) |
555 | { | |
ed8af2e4 | 556 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
557 | return arch_atomic_fetch_or_relaxed(i, v); |
558 | } | |
559 | #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed | |
560 | #endif | |
561 | ||
c020395b | 562 | static __always_inline void |
aa525d06 MR |
563 | atomic_xor(int i, atomic_t *v) |
564 | { | |
ed8af2e4 | 565 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
566 | arch_atomic_xor(i, v); |
567 | } | |
568 | #define atomic_xor atomic_xor | |
569 | ||
570 | #if !defined(arch_atomic_fetch_xor_relaxed) || defined(arch_atomic_fetch_xor) | |
c020395b | 571 | static __always_inline int |
aa525d06 MR |
572 | atomic_fetch_xor(int i, atomic_t *v) |
573 | { | |
ed8af2e4 | 574 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
575 | return arch_atomic_fetch_xor(i, v); |
576 | } | |
577 | #define atomic_fetch_xor atomic_fetch_xor | |
578 | #endif | |
579 | ||
580 | #if defined(arch_atomic_fetch_xor_acquire) | |
c020395b | 581 | static __always_inline int |
aa525d06 MR |
582 | atomic_fetch_xor_acquire(int i, atomic_t *v) |
583 | { | |
ed8af2e4 | 584 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
585 | return arch_atomic_fetch_xor_acquire(i, v); |
586 | } | |
587 | #define atomic_fetch_xor_acquire atomic_fetch_xor_acquire | |
588 | #endif | |
589 | ||
590 | #if defined(arch_atomic_fetch_xor_release) | |
c020395b | 591 | static __always_inline int |
aa525d06 MR |
592 | atomic_fetch_xor_release(int i, atomic_t *v) |
593 | { | |
ed8af2e4 | 594 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
595 | return arch_atomic_fetch_xor_release(i, v); |
596 | } | |
597 | #define atomic_fetch_xor_release atomic_fetch_xor_release | |
598 | #endif | |
599 | ||
600 | #if defined(arch_atomic_fetch_xor_relaxed) | |
c020395b | 601 | static __always_inline int |
aa525d06 MR |
602 | atomic_fetch_xor_relaxed(int i, atomic_t *v) |
603 | { | |
ed8af2e4 | 604 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
605 | return arch_atomic_fetch_xor_relaxed(i, v); |
606 | } | |
607 | #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed | |
608 | #endif | |
609 | ||
610 | #if !defined(arch_atomic_xchg_relaxed) || defined(arch_atomic_xchg) | |
c020395b | 611 | static __always_inline int |
aa525d06 MR |
612 | atomic_xchg(atomic_t *v, int i) |
613 | { | |
ed8af2e4 | 614 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
615 | return arch_atomic_xchg(v, i); |
616 | } | |
617 | #define atomic_xchg atomic_xchg | |
618 | #endif | |
619 | ||
620 | #if defined(arch_atomic_xchg_acquire) | |
c020395b | 621 | static __always_inline int |
aa525d06 MR |
622 | atomic_xchg_acquire(atomic_t *v, int i) |
623 | { | |
ed8af2e4 | 624 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
625 | return arch_atomic_xchg_acquire(v, i); |
626 | } | |
627 | #define atomic_xchg_acquire atomic_xchg_acquire | |
628 | #endif | |
629 | ||
630 | #if defined(arch_atomic_xchg_release) | |
c020395b | 631 | static __always_inline int |
aa525d06 MR |
632 | atomic_xchg_release(atomic_t *v, int i) |
633 | { | |
ed8af2e4 | 634 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
635 | return arch_atomic_xchg_release(v, i); |
636 | } | |
637 | #define atomic_xchg_release atomic_xchg_release | |
638 | #endif | |
639 | ||
640 | #if defined(arch_atomic_xchg_relaxed) | |
c020395b | 641 | static __always_inline int |
aa525d06 MR |
642 | atomic_xchg_relaxed(atomic_t *v, int i) |
643 | { | |
ed8af2e4 | 644 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
645 | return arch_atomic_xchg_relaxed(v, i); |
646 | } | |
647 | #define atomic_xchg_relaxed atomic_xchg_relaxed | |
648 | #endif | |
649 | ||
650 | #if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg) | |
c020395b | 651 | static __always_inline int |
aa525d06 MR |
652 | atomic_cmpxchg(atomic_t *v, int old, int new) |
653 | { | |
ed8af2e4 | 654 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
655 | return arch_atomic_cmpxchg(v, old, new); |
656 | } | |
657 | #define atomic_cmpxchg atomic_cmpxchg | |
658 | #endif | |
659 | ||
660 | #if defined(arch_atomic_cmpxchg_acquire) | |
c020395b | 661 | static __always_inline int |
aa525d06 MR |
662 | atomic_cmpxchg_acquire(atomic_t *v, int old, int new) |
663 | { | |
ed8af2e4 | 664 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
665 | return arch_atomic_cmpxchg_acquire(v, old, new); |
666 | } | |
667 | #define atomic_cmpxchg_acquire atomic_cmpxchg_acquire | |
668 | #endif | |
669 | ||
670 | #if defined(arch_atomic_cmpxchg_release) | |
c020395b | 671 | static __always_inline int |
aa525d06 MR |
672 | atomic_cmpxchg_release(atomic_t *v, int old, int new) |
673 | { | |
ed8af2e4 | 674 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
675 | return arch_atomic_cmpxchg_release(v, old, new); |
676 | } | |
677 | #define atomic_cmpxchg_release atomic_cmpxchg_release | |
678 | #endif | |
679 | ||
680 | #if defined(arch_atomic_cmpxchg_relaxed) | |
c020395b | 681 | static __always_inline int |
aa525d06 MR |
682 | atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) |
683 | { | |
ed8af2e4 | 684 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
685 | return arch_atomic_cmpxchg_relaxed(v, old, new); |
686 | } | |
687 | #define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed | |
688 | #endif | |
689 | ||
690 | #if defined(arch_atomic_try_cmpxchg) | |
c020395b | 691 | static __always_inline bool |
aa525d06 MR |
692 | atomic_try_cmpxchg(atomic_t *v, int *old, int new) |
693 | { | |
ed8af2e4 ME |
694 | instrument_atomic_write(v, sizeof(*v)); |
695 | instrument_atomic_write(old, sizeof(*old)); | |
aa525d06 MR |
696 | return arch_atomic_try_cmpxchg(v, old, new); |
697 | } | |
698 | #define atomic_try_cmpxchg atomic_try_cmpxchg | |
699 | #endif | |
700 | ||
701 | #if defined(arch_atomic_try_cmpxchg_acquire) | |
c020395b | 702 | static __always_inline bool |
aa525d06 MR |
703 | atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) |
704 | { | |
ed8af2e4 ME |
705 | instrument_atomic_write(v, sizeof(*v)); |
706 | instrument_atomic_write(old, sizeof(*old)); | |
aa525d06 MR |
707 | return arch_atomic_try_cmpxchg_acquire(v, old, new); |
708 | } | |
709 | #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire | |
710 | #endif | |
711 | ||
712 | #if defined(arch_atomic_try_cmpxchg_release) | |
c020395b | 713 | static __always_inline bool |
aa525d06 MR |
714 | atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) |
715 | { | |
ed8af2e4 ME |
716 | instrument_atomic_write(v, sizeof(*v)); |
717 | instrument_atomic_write(old, sizeof(*old)); | |
aa525d06 MR |
718 | return arch_atomic_try_cmpxchg_release(v, old, new); |
719 | } | |
720 | #define atomic_try_cmpxchg_release atomic_try_cmpxchg_release | |
721 | #endif | |
722 | ||
723 | #if defined(arch_atomic_try_cmpxchg_relaxed) | |
c020395b | 724 | static __always_inline bool |
aa525d06 MR |
725 | atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) |
726 | { | |
ed8af2e4 ME |
727 | instrument_atomic_write(v, sizeof(*v)); |
728 | instrument_atomic_write(old, sizeof(*old)); | |
aa525d06 MR |
729 | return arch_atomic_try_cmpxchg_relaxed(v, old, new); |
730 | } | |
731 | #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed | |
732 | #endif | |
733 | ||
734 | #if defined(arch_atomic_sub_and_test) | |
c020395b | 735 | static __always_inline bool |
aa525d06 MR |
736 | atomic_sub_and_test(int i, atomic_t *v) |
737 | { | |
ed8af2e4 | 738 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
739 | return arch_atomic_sub_and_test(i, v); |
740 | } | |
741 | #define atomic_sub_and_test atomic_sub_and_test | |
742 | #endif | |
743 | ||
744 | #if defined(arch_atomic_dec_and_test) | |
c020395b | 745 | static __always_inline bool |
aa525d06 MR |
746 | atomic_dec_and_test(atomic_t *v) |
747 | { | |
ed8af2e4 | 748 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
749 | return arch_atomic_dec_and_test(v); |
750 | } | |
751 | #define atomic_dec_and_test atomic_dec_and_test | |
752 | #endif | |
753 | ||
754 | #if defined(arch_atomic_inc_and_test) | |
c020395b | 755 | static __always_inline bool |
aa525d06 MR |
756 | atomic_inc_and_test(atomic_t *v) |
757 | { | |
ed8af2e4 | 758 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
759 | return arch_atomic_inc_and_test(v); |
760 | } | |
761 | #define atomic_inc_and_test atomic_inc_and_test | |
762 | #endif | |
763 | ||
764 | #if defined(arch_atomic_add_negative) | |
c020395b | 765 | static __always_inline bool |
aa525d06 MR |
766 | atomic_add_negative(int i, atomic_t *v) |
767 | { | |
ed8af2e4 | 768 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
769 | return arch_atomic_add_negative(i, v); |
770 | } | |
771 | #define atomic_add_negative atomic_add_negative | |
772 | #endif | |
773 | ||
774 | #if defined(arch_atomic_fetch_add_unless) | |
c020395b | 775 | static __always_inline int |
aa525d06 MR |
776 | atomic_fetch_add_unless(atomic_t *v, int a, int u) |
777 | { | |
ed8af2e4 | 778 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
779 | return arch_atomic_fetch_add_unless(v, a, u); |
780 | } | |
781 | #define atomic_fetch_add_unless atomic_fetch_add_unless | |
782 | #endif | |
783 | ||
784 | #if defined(arch_atomic_add_unless) | |
c020395b | 785 | static __always_inline bool |
aa525d06 MR |
786 | atomic_add_unless(atomic_t *v, int a, int u) |
787 | { | |
ed8af2e4 | 788 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
789 | return arch_atomic_add_unless(v, a, u); |
790 | } | |
791 | #define atomic_add_unless atomic_add_unless | |
792 | #endif | |
793 | ||
794 | #if defined(arch_atomic_inc_not_zero) | |
c020395b | 795 | static __always_inline bool |
aa525d06 MR |
796 | atomic_inc_not_zero(atomic_t *v) |
797 | { | |
ed8af2e4 | 798 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
799 | return arch_atomic_inc_not_zero(v); |
800 | } | |
801 | #define atomic_inc_not_zero atomic_inc_not_zero | |
802 | #endif | |
803 | ||
804 | #if defined(arch_atomic_inc_unless_negative) | |
c020395b | 805 | static __always_inline bool |
aa525d06 MR |
806 | atomic_inc_unless_negative(atomic_t *v) |
807 | { | |
ed8af2e4 | 808 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
809 | return arch_atomic_inc_unless_negative(v); |
810 | } | |
811 | #define atomic_inc_unless_negative atomic_inc_unless_negative | |
812 | #endif | |
813 | ||
814 | #if defined(arch_atomic_dec_unless_positive) | |
c020395b | 815 | static __always_inline bool |
aa525d06 MR |
816 | atomic_dec_unless_positive(atomic_t *v) |
817 | { | |
ed8af2e4 | 818 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
819 | return arch_atomic_dec_unless_positive(v); |
820 | } | |
821 | #define atomic_dec_unless_positive atomic_dec_unless_positive | |
822 | #endif | |
823 | ||
824 | #if defined(arch_atomic_dec_if_positive) | |
c020395b | 825 | static __always_inline int |
aa525d06 MR |
826 | atomic_dec_if_positive(atomic_t *v) |
827 | { | |
ed8af2e4 | 828 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
829 | return arch_atomic_dec_if_positive(v); |
830 | } | |
831 | #define atomic_dec_if_positive atomic_dec_if_positive | |
832 | #endif | |
833 | ||
c020395b | 834 | static __always_inline s64 |
aa525d06 MR |
835 | atomic64_read(const atomic64_t *v) |
836 | { | |
ed8af2e4 | 837 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
838 | return arch_atomic64_read(v); |
839 | } | |
840 | #define atomic64_read atomic64_read | |
841 | ||
842 | #if defined(arch_atomic64_read_acquire) | |
c020395b | 843 | static __always_inline s64 |
aa525d06 MR |
844 | atomic64_read_acquire(const atomic64_t *v) |
845 | { | |
ed8af2e4 | 846 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
847 | return arch_atomic64_read_acquire(v); |
848 | } | |
849 | #define atomic64_read_acquire atomic64_read_acquire | |
850 | #endif | |
851 | ||
c020395b | 852 | static __always_inline void |
aa525d06 MR |
853 | atomic64_set(atomic64_t *v, s64 i) |
854 | { | |
ed8af2e4 | 855 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
856 | arch_atomic64_set(v, i); |
857 | } | |
858 | #define atomic64_set atomic64_set | |
859 | ||
860 | #if defined(arch_atomic64_set_release) | |
c020395b | 861 | static __always_inline void |
aa525d06 MR |
862 | atomic64_set_release(atomic64_t *v, s64 i) |
863 | { | |
ed8af2e4 | 864 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
865 | arch_atomic64_set_release(v, i); |
866 | } | |
867 | #define atomic64_set_release atomic64_set_release | |
868 | #endif | |
869 | ||
c020395b | 870 | static __always_inline void |
aa525d06 MR |
871 | atomic64_add(s64 i, atomic64_t *v) |
872 | { | |
ed8af2e4 | 873 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
874 | arch_atomic64_add(i, v); |
875 | } | |
876 | #define atomic64_add atomic64_add | |
877 | ||
878 | #if !defined(arch_atomic64_add_return_relaxed) || defined(arch_atomic64_add_return) | |
c020395b | 879 | static __always_inline s64 |
aa525d06 MR |
880 | atomic64_add_return(s64 i, atomic64_t *v) |
881 | { | |
ed8af2e4 | 882 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
883 | return arch_atomic64_add_return(i, v); |
884 | } | |
885 | #define atomic64_add_return atomic64_add_return | |
886 | #endif | |
887 | ||
888 | #if defined(arch_atomic64_add_return_acquire) | |
c020395b | 889 | static __always_inline s64 |
aa525d06 MR |
890 | atomic64_add_return_acquire(s64 i, atomic64_t *v) |
891 | { | |
ed8af2e4 | 892 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
893 | return arch_atomic64_add_return_acquire(i, v); |
894 | } | |
895 | #define atomic64_add_return_acquire atomic64_add_return_acquire | |
896 | #endif | |
897 | ||
898 | #if defined(arch_atomic64_add_return_release) | |
c020395b | 899 | static __always_inline s64 |
aa525d06 MR |
900 | atomic64_add_return_release(s64 i, atomic64_t *v) |
901 | { | |
ed8af2e4 | 902 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
903 | return arch_atomic64_add_return_release(i, v); |
904 | } | |
905 | #define atomic64_add_return_release atomic64_add_return_release | |
906 | #endif | |
907 | ||
908 | #if defined(arch_atomic64_add_return_relaxed) | |
c020395b | 909 | static __always_inline s64 |
aa525d06 MR |
910 | atomic64_add_return_relaxed(s64 i, atomic64_t *v) |
911 | { | |
ed8af2e4 | 912 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
913 | return arch_atomic64_add_return_relaxed(i, v); |
914 | } | |
915 | #define atomic64_add_return_relaxed atomic64_add_return_relaxed | |
916 | #endif | |
917 | ||
918 | #if !defined(arch_atomic64_fetch_add_relaxed) || defined(arch_atomic64_fetch_add) | |
c020395b | 919 | static __always_inline s64 |
aa525d06 MR |
920 | atomic64_fetch_add(s64 i, atomic64_t *v) |
921 | { | |
ed8af2e4 | 922 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
923 | return arch_atomic64_fetch_add(i, v); |
924 | } | |
925 | #define atomic64_fetch_add atomic64_fetch_add | |
926 | #endif | |
927 | ||
928 | #if defined(arch_atomic64_fetch_add_acquire) | |
c020395b | 929 | static __always_inline s64 |
aa525d06 MR |
930 | atomic64_fetch_add_acquire(s64 i, atomic64_t *v) |
931 | { | |
ed8af2e4 | 932 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
933 | return arch_atomic64_fetch_add_acquire(i, v); |
934 | } | |
935 | #define atomic64_fetch_add_acquire atomic64_fetch_add_acquire | |
936 | #endif | |
937 | ||
938 | #if defined(arch_atomic64_fetch_add_release) | |
c020395b | 939 | static __always_inline s64 |
aa525d06 MR |
940 | atomic64_fetch_add_release(s64 i, atomic64_t *v) |
941 | { | |
ed8af2e4 | 942 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
943 | return arch_atomic64_fetch_add_release(i, v); |
944 | } | |
945 | #define atomic64_fetch_add_release atomic64_fetch_add_release | |
946 | #endif | |
947 | ||
948 | #if defined(arch_atomic64_fetch_add_relaxed) | |
c020395b | 949 | static __always_inline s64 |
aa525d06 MR |
950 | atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) |
951 | { | |
ed8af2e4 | 952 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
953 | return arch_atomic64_fetch_add_relaxed(i, v); |
954 | } | |
955 | #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed | |
956 | #endif | |
957 | ||
c020395b | 958 | static __always_inline void |
aa525d06 MR |
959 | atomic64_sub(s64 i, atomic64_t *v) |
960 | { | |
ed8af2e4 | 961 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
962 | arch_atomic64_sub(i, v); |
963 | } | |
964 | #define atomic64_sub atomic64_sub | |
ac605bee | 965 | |
aa525d06 | 966 | #if !defined(arch_atomic64_sub_return_relaxed) || defined(arch_atomic64_sub_return) |
c020395b | 967 | static __always_inline s64 |
aa525d06 MR |
968 | atomic64_sub_return(s64 i, atomic64_t *v) |
969 | { | |
ed8af2e4 | 970 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
971 | return arch_atomic64_sub_return(i, v); |
972 | } | |
973 | #define atomic64_sub_return atomic64_sub_return | |
974 | #endif | |
b06ed71a | 975 | |
aa525d06 | 976 | #if defined(arch_atomic64_sub_return_acquire) |
c020395b | 977 | static __always_inline s64 |
aa525d06 MR |
978 | atomic64_sub_return_acquire(s64 i, atomic64_t *v) |
979 | { | |
ed8af2e4 | 980 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
981 | return arch_atomic64_sub_return_acquire(i, v); |
982 | } | |
983 | #define atomic64_sub_return_acquire atomic64_sub_return_acquire | |
984 | #endif | |
b06ed71a | 985 | |
aa525d06 | 986 | #if defined(arch_atomic64_sub_return_release) |
c020395b | 987 | static __always_inline s64 |
aa525d06 | 988 | atomic64_sub_return_release(s64 i, atomic64_t *v) |
b06ed71a | 989 | { |
ed8af2e4 | 990 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 991 | return arch_atomic64_sub_return_release(i, v); |
b06ed71a | 992 | } |
aa525d06 MR |
993 | #define atomic64_sub_return_release atomic64_sub_return_release |
994 | #endif | |
b06ed71a | 995 | |
aa525d06 | 996 | #if defined(arch_atomic64_sub_return_relaxed) |
c020395b | 997 | static __always_inline s64 |
aa525d06 | 998 | atomic64_sub_return_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 999 | { |
ed8af2e4 | 1000 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1001 | return arch_atomic64_sub_return_relaxed(i, v); |
b06ed71a | 1002 | } |
aa525d06 MR |
1003 | #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed |
1004 | #endif | |
b06ed71a | 1005 | |
aa525d06 | 1006 | #if !defined(arch_atomic64_fetch_sub_relaxed) || defined(arch_atomic64_fetch_sub) |
c020395b | 1007 | static __always_inline s64 |
aa525d06 | 1008 | atomic64_fetch_sub(s64 i, atomic64_t *v) |
b06ed71a | 1009 | { |
ed8af2e4 | 1010 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1011 | return arch_atomic64_fetch_sub(i, v); |
b06ed71a | 1012 | } |
aa525d06 MR |
1013 | #define atomic64_fetch_sub atomic64_fetch_sub |
1014 | #endif | |
b06ed71a | 1015 | |
aa525d06 | 1016 | #if defined(arch_atomic64_fetch_sub_acquire) |
c020395b | 1017 | static __always_inline s64 |
aa525d06 | 1018 | atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) |
b06ed71a | 1019 | { |
ed8af2e4 | 1020 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1021 | return arch_atomic64_fetch_sub_acquire(i, v); |
b06ed71a | 1022 | } |
aa525d06 MR |
1023 | #define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire |
1024 | #endif | |
b06ed71a | 1025 | |
aa525d06 | 1026 | #if defined(arch_atomic64_fetch_sub_release) |
c020395b | 1027 | static __always_inline s64 |
aa525d06 | 1028 | atomic64_fetch_sub_release(s64 i, atomic64_t *v) |
b06ed71a | 1029 | { |
ed8af2e4 | 1030 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1031 | return arch_atomic64_fetch_sub_release(i, v); |
b06ed71a | 1032 | } |
aa525d06 MR |
1033 | #define atomic64_fetch_sub_release atomic64_fetch_sub_release |
1034 | #endif | |
b06ed71a | 1035 | |
aa525d06 | 1036 | #if defined(arch_atomic64_fetch_sub_relaxed) |
c020395b | 1037 | static __always_inline s64 |
aa525d06 | 1038 | atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 1039 | { |
ed8af2e4 | 1040 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1041 | return arch_atomic64_fetch_sub_relaxed(i, v); |
b06ed71a | 1042 | } |
aa525d06 MR |
1043 | #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed |
1044 | #endif | |
b06ed71a | 1045 | |
aa525d06 | 1046 | #if defined(arch_atomic64_inc) |
c020395b | 1047 | static __always_inline void |
aa525d06 | 1048 | atomic64_inc(atomic64_t *v) |
b06ed71a | 1049 | { |
ed8af2e4 | 1050 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1051 | arch_atomic64_inc(v); |
b06ed71a | 1052 | } |
aa525d06 MR |
1053 | #define atomic64_inc atomic64_inc |
1054 | #endif | |
b06ed71a | 1055 | |
aa525d06 | 1056 | #if defined(arch_atomic64_inc_return) |
c020395b | 1057 | static __always_inline s64 |
aa525d06 | 1058 | atomic64_inc_return(atomic64_t *v) |
b06ed71a | 1059 | { |
ed8af2e4 | 1060 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1061 | return arch_atomic64_inc_return(v); |
b06ed71a | 1062 | } |
aa525d06 MR |
1063 | #define atomic64_inc_return atomic64_inc_return |
1064 | #endif | |
b06ed71a | 1065 | |
aa525d06 | 1066 | #if defined(arch_atomic64_inc_return_acquire) |
c020395b | 1067 | static __always_inline s64 |
aa525d06 | 1068 | atomic64_inc_return_acquire(atomic64_t *v) |
b06ed71a | 1069 | { |
ed8af2e4 | 1070 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1071 | return arch_atomic64_inc_return_acquire(v); |
b06ed71a | 1072 | } |
aa525d06 | 1073 | #define atomic64_inc_return_acquire atomic64_inc_return_acquire |
b06ed71a DV |
1074 | #endif |
1075 | ||
aa525d06 | 1076 | #if defined(arch_atomic64_inc_return_release) |
c020395b | 1077 | static __always_inline s64 |
aa525d06 | 1078 | atomic64_inc_return_release(atomic64_t *v) |
b06ed71a | 1079 | { |
ed8af2e4 | 1080 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1081 | return arch_atomic64_inc_return_release(v); |
b06ed71a | 1082 | } |
aa525d06 | 1083 | #define atomic64_inc_return_release atomic64_inc_return_release |
b06ed71a DV |
1084 | #endif |
1085 | ||
aa525d06 | 1086 | #if defined(arch_atomic64_inc_return_relaxed) |
c020395b | 1087 | static __always_inline s64 |
aa525d06 | 1088 | atomic64_inc_return_relaxed(atomic64_t *v) |
b06ed71a | 1089 | { |
ed8af2e4 | 1090 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1091 | return arch_atomic64_inc_return_relaxed(v); |
b06ed71a | 1092 | } |
aa525d06 | 1093 | #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed |
eccc2da8 | 1094 | #endif |
b06ed71a | 1095 | |
aa525d06 | 1096 | #if defined(arch_atomic64_fetch_inc) |
c020395b | 1097 | static __always_inline s64 |
aa525d06 | 1098 | atomic64_fetch_inc(atomic64_t *v) |
0ae1d994 | 1099 | { |
ed8af2e4 | 1100 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1101 | return arch_atomic64_fetch_inc(v); |
0ae1d994 | 1102 | } |
aa525d06 | 1103 | #define atomic64_fetch_inc atomic64_fetch_inc |
0ae1d994 | 1104 | #endif |
b06ed71a | 1105 | |
aa525d06 | 1106 | #if defined(arch_atomic64_fetch_inc_acquire) |
c020395b | 1107 | static __always_inline s64 |
aa525d06 | 1108 | atomic64_fetch_inc_acquire(atomic64_t *v) |
b06ed71a | 1109 | { |
ed8af2e4 | 1110 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1111 | return arch_atomic64_fetch_inc_acquire(v); |
b06ed71a | 1112 | } |
aa525d06 | 1113 | #define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire |
9837559d | 1114 | #endif |
b06ed71a | 1115 | |
aa525d06 | 1116 | #if defined(arch_atomic64_fetch_inc_release) |
c020395b | 1117 | static __always_inline s64 |
aa525d06 | 1118 | atomic64_fetch_inc_release(atomic64_t *v) |
b06ed71a | 1119 | { |
ed8af2e4 | 1120 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1121 | return arch_atomic64_fetch_inc_release(v); |
b06ed71a | 1122 | } |
aa525d06 | 1123 | #define atomic64_fetch_inc_release atomic64_fetch_inc_release |
9837559d | 1124 | #endif |
b06ed71a | 1125 | |
aa525d06 | 1126 | #if defined(arch_atomic64_fetch_inc_relaxed) |
c020395b | 1127 | static __always_inline s64 |
aa525d06 | 1128 | atomic64_fetch_inc_relaxed(atomic64_t *v) |
b06ed71a | 1129 | { |
ed8af2e4 | 1130 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1131 | return arch_atomic64_fetch_inc_relaxed(v); |
b06ed71a | 1132 | } |
aa525d06 | 1133 | #define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed |
9837559d | 1134 | #endif |
b06ed71a | 1135 | |
aa525d06 | 1136 | #if defined(arch_atomic64_dec) |
c020395b | 1137 | static __always_inline void |
aa525d06 | 1138 | atomic64_dec(atomic64_t *v) |
b06ed71a | 1139 | { |
ed8af2e4 | 1140 | instrument_atomic_write(v, sizeof(*v)); |
b06ed71a DV |
1141 | arch_atomic64_dec(v); |
1142 | } | |
aa525d06 | 1143 | #define atomic64_dec atomic64_dec |
9837559d | 1144 | #endif |
b06ed71a | 1145 | |
aa525d06 | 1146 | #if defined(arch_atomic64_dec_return) |
c020395b | 1147 | static __always_inline s64 |
aa525d06 | 1148 | atomic64_dec_return(atomic64_t *v) |
b06ed71a | 1149 | { |
ed8af2e4 | 1150 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1151 | return arch_atomic64_dec_return(v); |
b06ed71a | 1152 | } |
aa525d06 MR |
1153 | #define atomic64_dec_return atomic64_dec_return |
1154 | #endif | |
b06ed71a | 1155 | |
aa525d06 | 1156 | #if defined(arch_atomic64_dec_return_acquire) |
c020395b | 1157 | static __always_inline s64 |
aa525d06 | 1158 | atomic64_dec_return_acquire(atomic64_t *v) |
b06ed71a | 1159 | { |
ed8af2e4 | 1160 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1161 | return arch_atomic64_dec_return_acquire(v); |
b06ed71a | 1162 | } |
aa525d06 MR |
1163 | #define atomic64_dec_return_acquire atomic64_dec_return_acquire |
1164 | #endif | |
b06ed71a | 1165 | |
aa525d06 | 1166 | #if defined(arch_atomic64_dec_return_release) |
c020395b | 1167 | static __always_inline s64 |
aa525d06 | 1168 | atomic64_dec_return_release(atomic64_t *v) |
b06ed71a | 1169 | { |
ed8af2e4 | 1170 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1171 | return arch_atomic64_dec_return_release(v); |
b06ed71a | 1172 | } |
aa525d06 MR |
1173 | #define atomic64_dec_return_release atomic64_dec_return_release |
1174 | #endif | |
b06ed71a | 1175 | |
aa525d06 | 1176 | #if defined(arch_atomic64_dec_return_relaxed) |
c020395b | 1177 | static __always_inline s64 |
aa525d06 | 1178 | atomic64_dec_return_relaxed(atomic64_t *v) |
b06ed71a | 1179 | { |
ed8af2e4 | 1180 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1181 | return arch_atomic64_dec_return_relaxed(v); |
b06ed71a | 1182 | } |
aa525d06 MR |
1183 | #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed |
1184 | #endif | |
b06ed71a | 1185 | |
aa525d06 | 1186 | #if defined(arch_atomic64_fetch_dec) |
c020395b | 1187 | static __always_inline s64 |
aa525d06 | 1188 | atomic64_fetch_dec(atomic64_t *v) |
b06ed71a | 1189 | { |
ed8af2e4 | 1190 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
1191 | return arch_atomic64_fetch_dec(v); |
1192 | } | |
1193 | #define atomic64_fetch_dec atomic64_fetch_dec | |
1194 | #endif | |
1195 | ||
1196 | #if defined(arch_atomic64_fetch_dec_acquire) | |
c020395b | 1197 | static __always_inline s64 |
aa525d06 MR |
1198 | atomic64_fetch_dec_acquire(atomic64_t *v) |
1199 | { | |
ed8af2e4 | 1200 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
1201 | return arch_atomic64_fetch_dec_acquire(v); |
1202 | } | |
1203 | #define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire | |
1204 | #endif | |
1205 | ||
1206 | #if defined(arch_atomic64_fetch_dec_release) | |
c020395b | 1207 | static __always_inline s64 |
aa525d06 MR |
1208 | atomic64_fetch_dec_release(atomic64_t *v) |
1209 | { | |
ed8af2e4 | 1210 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1211 | return arch_atomic64_fetch_dec_release(v); |
b06ed71a | 1212 | } |
aa525d06 MR |
1213 | #define atomic64_fetch_dec_release atomic64_fetch_dec_release |
1214 | #endif | |
1215 | ||
1216 | #if defined(arch_atomic64_fetch_dec_relaxed) | |
c020395b | 1217 | static __always_inline s64 |
aa525d06 MR |
1218 | atomic64_fetch_dec_relaxed(atomic64_t *v) |
1219 | { | |
ed8af2e4 | 1220 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
1221 | return arch_atomic64_fetch_dec_relaxed(v); |
1222 | } | |
1223 | #define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed | |
1224 | #endif | |
b06ed71a | 1225 | |
c020395b | 1226 | static __always_inline void |
aa525d06 | 1227 | atomic64_and(s64 i, atomic64_t *v) |
b06ed71a | 1228 | { |
ed8af2e4 | 1229 | instrument_atomic_write(v, sizeof(*v)); |
b06ed71a DV |
1230 | arch_atomic64_and(i, v); |
1231 | } | |
aa525d06 | 1232 | #define atomic64_and atomic64_and |
b06ed71a | 1233 | |
aa525d06 | 1234 | #if !defined(arch_atomic64_fetch_and_relaxed) || defined(arch_atomic64_fetch_and) |
c020395b | 1235 | static __always_inline s64 |
aa525d06 | 1236 | atomic64_fetch_and(s64 i, atomic64_t *v) |
b06ed71a | 1237 | { |
ed8af2e4 | 1238 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1239 | return arch_atomic64_fetch_and(i, v); |
b06ed71a | 1240 | } |
aa525d06 MR |
1241 | #define atomic64_fetch_and atomic64_fetch_and |
1242 | #endif | |
b06ed71a | 1243 | |
aa525d06 | 1244 | #if defined(arch_atomic64_fetch_and_acquire) |
c020395b | 1245 | static __always_inline s64 |
aa525d06 | 1246 | atomic64_fetch_and_acquire(s64 i, atomic64_t *v) |
b06ed71a | 1247 | { |
ed8af2e4 | 1248 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1249 | return arch_atomic64_fetch_and_acquire(i, v); |
b06ed71a | 1250 | } |
aa525d06 MR |
1251 | #define atomic64_fetch_and_acquire atomic64_fetch_and_acquire |
1252 | #endif | |
b06ed71a | 1253 | |
aa525d06 | 1254 | #if defined(arch_atomic64_fetch_and_release) |
c020395b | 1255 | static __always_inline s64 |
aa525d06 | 1256 | atomic64_fetch_and_release(s64 i, atomic64_t *v) |
b06ed71a | 1257 | { |
ed8af2e4 | 1258 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1259 | return arch_atomic64_fetch_and_release(i, v); |
b06ed71a | 1260 | } |
aa525d06 MR |
1261 | #define atomic64_fetch_and_release atomic64_fetch_and_release |
1262 | #endif | |
b06ed71a | 1263 | |
aa525d06 | 1264 | #if defined(arch_atomic64_fetch_and_relaxed) |
c020395b | 1265 | static __always_inline s64 |
aa525d06 | 1266 | atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 1267 | { |
ed8af2e4 | 1268 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1269 | return arch_atomic64_fetch_and_relaxed(i, v); |
b06ed71a | 1270 | } |
aa525d06 MR |
1271 | #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed |
1272 | #endif | |
b06ed71a | 1273 | |
aa525d06 | 1274 | #if defined(arch_atomic64_andnot) |
c020395b | 1275 | static __always_inline void |
aa525d06 | 1276 | atomic64_andnot(s64 i, atomic64_t *v) |
b06ed71a | 1277 | { |
ed8af2e4 | 1278 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1279 | arch_atomic64_andnot(i, v); |
b06ed71a | 1280 | } |
aa525d06 | 1281 | #define atomic64_andnot atomic64_andnot |
9837559d | 1282 | #endif |
b06ed71a | 1283 | |
aa525d06 | 1284 | #if defined(arch_atomic64_fetch_andnot) |
c020395b | 1285 | static __always_inline s64 |
aa525d06 | 1286 | atomic64_fetch_andnot(s64 i, atomic64_t *v) |
b06ed71a | 1287 | { |
ed8af2e4 | 1288 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1289 | return arch_atomic64_fetch_andnot(i, v); |
b06ed71a | 1290 | } |
aa525d06 | 1291 | #define atomic64_fetch_andnot atomic64_fetch_andnot |
9837559d | 1292 | #endif |
b06ed71a | 1293 | |
aa525d06 | 1294 | #if defined(arch_atomic64_fetch_andnot_acquire) |
c020395b | 1295 | static __always_inline s64 |
aa525d06 | 1296 | atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) |
b06ed71a | 1297 | { |
ed8af2e4 | 1298 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1299 | return arch_atomic64_fetch_andnot_acquire(i, v); |
b06ed71a | 1300 | } |
aa525d06 | 1301 | #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire |
9837559d | 1302 | #endif |
b06ed71a | 1303 | |
aa525d06 | 1304 | #if defined(arch_atomic64_fetch_andnot_release) |
c020395b | 1305 | static __always_inline s64 |
aa525d06 | 1306 | atomic64_fetch_andnot_release(s64 i, atomic64_t *v) |
b06ed71a | 1307 | { |
ed8af2e4 | 1308 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1309 | return arch_atomic64_fetch_andnot_release(i, v); |
b06ed71a | 1310 | } |
aa525d06 | 1311 | #define atomic64_fetch_andnot_release atomic64_fetch_andnot_release |
9837559d | 1312 | #endif |
b06ed71a | 1313 | |
aa525d06 | 1314 | #if defined(arch_atomic64_fetch_andnot_relaxed) |
c020395b | 1315 | static __always_inline s64 |
aa525d06 | 1316 | atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 1317 | { |
ed8af2e4 | 1318 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1319 | return arch_atomic64_fetch_andnot_relaxed(i, v); |
b06ed71a | 1320 | } |
aa525d06 | 1321 | #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed |
bef82820 | 1322 | #endif |
b06ed71a | 1323 | |
c020395b | 1324 | static __always_inline void |
aa525d06 | 1325 | atomic64_or(s64 i, atomic64_t *v) |
b06ed71a | 1326 | { |
ed8af2e4 | 1327 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
1328 | arch_atomic64_or(i, v); |
1329 | } | |
1330 | #define atomic64_or atomic64_or | |
1331 | ||
1332 | #if !defined(arch_atomic64_fetch_or_relaxed) || defined(arch_atomic64_fetch_or) | |
c020395b | 1333 | static __always_inline s64 |
aa525d06 MR |
1334 | atomic64_fetch_or(s64 i, atomic64_t *v) |
1335 | { | |
ed8af2e4 | 1336 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1337 | return arch_atomic64_fetch_or(i, v); |
b06ed71a | 1338 | } |
aa525d06 | 1339 | #define atomic64_fetch_or atomic64_fetch_or |
b3a2a05f | 1340 | #endif |
b06ed71a | 1341 | |
aa525d06 | 1342 | #if defined(arch_atomic64_fetch_or_acquire) |
c020395b | 1343 | static __always_inline s64 |
aa525d06 | 1344 | atomic64_fetch_or_acquire(s64 i, atomic64_t *v) |
b06ed71a | 1345 | { |
ed8af2e4 | 1346 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1347 | return arch_atomic64_fetch_or_acquire(i, v); |
b06ed71a | 1348 | } |
aa525d06 | 1349 | #define atomic64_fetch_or_acquire atomic64_fetch_or_acquire |
18cc1814 | 1350 | #endif |
b06ed71a | 1351 | |
aa525d06 | 1352 | #if defined(arch_atomic64_fetch_or_release) |
c020395b | 1353 | static __always_inline s64 |
aa525d06 | 1354 | atomic64_fetch_or_release(s64 i, atomic64_t *v) |
b06ed71a | 1355 | { |
ed8af2e4 | 1356 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1357 | return arch_atomic64_fetch_or_release(i, v); |
b06ed71a | 1358 | } |
aa525d06 | 1359 | #define atomic64_fetch_or_release atomic64_fetch_or_release |
18cc1814 | 1360 | #endif |
b06ed71a | 1361 | |
aa525d06 | 1362 | #if defined(arch_atomic64_fetch_or_relaxed) |
c020395b | 1363 | static __always_inline s64 |
aa525d06 | 1364 | atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 1365 | { |
ed8af2e4 | 1366 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1367 | return arch_atomic64_fetch_or_relaxed(i, v); |
b06ed71a | 1368 | } |
aa525d06 | 1369 | #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed |
18cc1814 | 1370 | #endif |
b06ed71a | 1371 | |
c020395b | 1372 | static __always_inline void |
aa525d06 | 1373 | atomic64_xor(s64 i, atomic64_t *v) |
b06ed71a | 1374 | { |
ed8af2e4 | 1375 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
1376 | arch_atomic64_xor(i, v); |
1377 | } | |
1378 | #define atomic64_xor atomic64_xor | |
1379 | ||
1380 | #if !defined(arch_atomic64_fetch_xor_relaxed) || defined(arch_atomic64_fetch_xor) | |
c020395b | 1381 | static __always_inline s64 |
aa525d06 MR |
1382 | atomic64_fetch_xor(s64 i, atomic64_t *v) |
1383 | { | |
ed8af2e4 | 1384 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1385 | return arch_atomic64_fetch_xor(i, v); |
b06ed71a | 1386 | } |
aa525d06 | 1387 | #define atomic64_fetch_xor atomic64_fetch_xor |
18cc1814 | 1388 | #endif |
b06ed71a | 1389 | |
aa525d06 | 1390 | #if defined(arch_atomic64_fetch_xor_acquire) |
c020395b | 1391 | static __always_inline s64 |
aa525d06 | 1392 | atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) |
b06ed71a | 1393 | { |
ed8af2e4 | 1394 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1395 | return arch_atomic64_fetch_xor_acquire(i, v); |
b06ed71a | 1396 | } |
aa525d06 MR |
1397 | #define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire |
1398 | #endif | |
b06ed71a | 1399 | |
aa525d06 | 1400 | #if defined(arch_atomic64_fetch_xor_release) |
c020395b | 1401 | static __always_inline s64 |
aa525d06 | 1402 | atomic64_fetch_xor_release(s64 i, atomic64_t *v) |
b06ed71a | 1403 | { |
ed8af2e4 | 1404 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1405 | return arch_atomic64_fetch_xor_release(i, v); |
b06ed71a | 1406 | } |
aa525d06 MR |
1407 | #define atomic64_fetch_xor_release atomic64_fetch_xor_release |
1408 | #endif | |
b06ed71a | 1409 | |
aa525d06 | 1410 | #if defined(arch_atomic64_fetch_xor_relaxed) |
c020395b | 1411 | static __always_inline s64 |
aa525d06 | 1412 | atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 1413 | { |
ed8af2e4 | 1414 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1415 | return arch_atomic64_fetch_xor_relaxed(i, v); |
b06ed71a | 1416 | } |
aa525d06 MR |
1417 | #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed |
1418 | #endif | |
b06ed71a | 1419 | |
aa525d06 | 1420 | #if !defined(arch_atomic64_xchg_relaxed) || defined(arch_atomic64_xchg) |
c020395b | 1421 | static __always_inline s64 |
aa525d06 | 1422 | atomic64_xchg(atomic64_t *v, s64 i) |
b06ed71a | 1423 | { |
ed8af2e4 | 1424 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1425 | return arch_atomic64_xchg(v, i); |
b06ed71a | 1426 | } |
aa525d06 MR |
1427 | #define atomic64_xchg atomic64_xchg |
1428 | #endif | |
b06ed71a | 1429 | |
aa525d06 | 1430 | #if defined(arch_atomic64_xchg_acquire) |
c020395b | 1431 | static __always_inline s64 |
aa525d06 | 1432 | atomic64_xchg_acquire(atomic64_t *v, s64 i) |
b06ed71a | 1433 | { |
ed8af2e4 | 1434 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1435 | return arch_atomic64_xchg_acquire(v, i); |
b06ed71a | 1436 | } |
aa525d06 MR |
1437 | #define atomic64_xchg_acquire atomic64_xchg_acquire |
1438 | #endif | |
b06ed71a | 1439 | |
aa525d06 | 1440 | #if defined(arch_atomic64_xchg_release) |
c020395b | 1441 | static __always_inline s64 |
aa525d06 | 1442 | atomic64_xchg_release(atomic64_t *v, s64 i) |
b06ed71a | 1443 | { |
ed8af2e4 | 1444 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1445 | return arch_atomic64_xchg_release(v, i); |
b06ed71a | 1446 | } |
aa525d06 MR |
1447 | #define atomic64_xchg_release atomic64_xchg_release |
1448 | #endif | |
b06ed71a | 1449 | |
aa525d06 | 1450 | #if defined(arch_atomic64_xchg_relaxed) |
c020395b | 1451 | static __always_inline s64 |
aa525d06 | 1452 | atomic64_xchg_relaxed(atomic64_t *v, s64 i) |
b06ed71a | 1453 | { |
ed8af2e4 | 1454 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1455 | return arch_atomic64_xchg_relaxed(v, i); |
b06ed71a | 1456 | } |
aa525d06 MR |
1457 | #define atomic64_xchg_relaxed atomic64_xchg_relaxed |
1458 | #endif | |
b06ed71a | 1459 | |
aa525d06 | 1460 | #if !defined(arch_atomic64_cmpxchg_relaxed) || defined(arch_atomic64_cmpxchg) |
c020395b | 1461 | static __always_inline s64 |
aa525d06 | 1462 | atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1463 | { |
ed8af2e4 | 1464 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1465 | return arch_atomic64_cmpxchg(v, old, new); |
b06ed71a | 1466 | } |
aa525d06 MR |
1467 | #define atomic64_cmpxchg atomic64_cmpxchg |
1468 | #endif | |
b06ed71a | 1469 | |
aa525d06 | 1470 | #if defined(arch_atomic64_cmpxchg_acquire) |
c020395b | 1471 | static __always_inline s64 |
aa525d06 | 1472 | atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1473 | { |
ed8af2e4 | 1474 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1475 | return arch_atomic64_cmpxchg_acquire(v, old, new); |
b06ed71a | 1476 | } |
aa525d06 MR |
1477 | #define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire |
1478 | #endif | |
b06ed71a | 1479 | |
aa525d06 | 1480 | #if defined(arch_atomic64_cmpxchg_release) |
c020395b | 1481 | static __always_inline s64 |
aa525d06 | 1482 | atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1483 | { |
ed8af2e4 | 1484 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1485 | return arch_atomic64_cmpxchg_release(v, old, new); |
b06ed71a | 1486 | } |
aa525d06 MR |
1487 | #define atomic64_cmpxchg_release atomic64_cmpxchg_release |
1488 | #endif | |
b06ed71a | 1489 | |
aa525d06 | 1490 | #if defined(arch_atomic64_cmpxchg_relaxed) |
c020395b | 1491 | static __always_inline s64 |
aa525d06 | 1492 | atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1493 | { |
ed8af2e4 | 1494 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1495 | return arch_atomic64_cmpxchg_relaxed(v, old, new); |
b06ed71a | 1496 | } |
aa525d06 MR |
1497 | #define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed |
1498 | #endif | |
b06ed71a | 1499 | |
aa525d06 | 1500 | #if defined(arch_atomic64_try_cmpxchg) |
c020395b | 1501 | static __always_inline bool |
aa525d06 | 1502 | atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1503 | { |
ed8af2e4 ME |
1504 | instrument_atomic_write(v, sizeof(*v)); |
1505 | instrument_atomic_write(old, sizeof(*old)); | |
aa525d06 | 1506 | return arch_atomic64_try_cmpxchg(v, old, new); |
b06ed71a | 1507 | } |
aa525d06 MR |
1508 | #define atomic64_try_cmpxchg atomic64_try_cmpxchg |
1509 | #endif | |
b06ed71a | 1510 | |
aa525d06 | 1511 | #if defined(arch_atomic64_try_cmpxchg_acquire) |
c020395b | 1512 | static __always_inline bool |
aa525d06 | 1513 | atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1514 | { |
ed8af2e4 ME |
1515 | instrument_atomic_write(v, sizeof(*v)); |
1516 | instrument_atomic_write(old, sizeof(*old)); | |
aa525d06 | 1517 | return arch_atomic64_try_cmpxchg_acquire(v, old, new); |
b06ed71a | 1518 | } |
aa525d06 MR |
1519 | #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire |
1520 | #endif | |
b06ed71a | 1521 | |
aa525d06 | 1522 | #if defined(arch_atomic64_try_cmpxchg_release) |
c020395b | 1523 | static __always_inline bool |
aa525d06 | 1524 | atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1525 | { |
ed8af2e4 ME |
1526 | instrument_atomic_write(v, sizeof(*v)); |
1527 | instrument_atomic_write(old, sizeof(*old)); | |
aa525d06 | 1528 | return arch_atomic64_try_cmpxchg_release(v, old, new); |
b06ed71a | 1529 | } |
aa525d06 MR |
1530 | #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release |
1531 | #endif | |
b06ed71a | 1532 | |
aa525d06 | 1533 | #if defined(arch_atomic64_try_cmpxchg_relaxed) |
c020395b | 1534 | static __always_inline bool |
aa525d06 | 1535 | atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1536 | { |
ed8af2e4 ME |
1537 | instrument_atomic_write(v, sizeof(*v)); |
1538 | instrument_atomic_write(old, sizeof(*old)); | |
aa525d06 | 1539 | return arch_atomic64_try_cmpxchg_relaxed(v, old, new); |
b06ed71a | 1540 | } |
aa525d06 | 1541 | #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed |
18cc1814 | 1542 | #endif |
b06ed71a | 1543 | |
aa525d06 | 1544 | #if defined(arch_atomic64_sub_and_test) |
c020395b | 1545 | static __always_inline bool |
aa525d06 | 1546 | atomic64_sub_and_test(s64 i, atomic64_t *v) |
b06ed71a | 1547 | { |
ed8af2e4 | 1548 | instrument_atomic_write(v, sizeof(*v)); |
b06ed71a DV |
1549 | return arch_atomic64_sub_and_test(i, v); |
1550 | } | |
aa525d06 | 1551 | #define atomic64_sub_and_test atomic64_sub_and_test |
18cc1814 | 1552 | #endif |
b06ed71a | 1553 | |
aa525d06 | 1554 | #if defined(arch_atomic64_dec_and_test) |
c020395b | 1555 | static __always_inline bool |
aa525d06 | 1556 | atomic64_dec_and_test(atomic64_t *v) |
b06ed71a | 1557 | { |
ed8af2e4 | 1558 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 | 1559 | return arch_atomic64_dec_and_test(v); |
b06ed71a | 1560 | } |
aa525d06 | 1561 | #define atomic64_dec_and_test atomic64_dec_and_test |
18cc1814 | 1562 | #endif |
b06ed71a | 1563 | |
aa525d06 | 1564 | #if defined(arch_atomic64_inc_and_test) |
c020395b | 1565 | static __always_inline bool |
aa525d06 MR |
1566 | atomic64_inc_and_test(atomic64_t *v) |
1567 | { | |
ed8af2e4 | 1568 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
1569 | return arch_atomic64_inc_and_test(v); |
1570 | } | |
1571 | #define atomic64_inc_and_test atomic64_inc_and_test | |
1572 | #endif | |
1573 | ||
1574 | #if defined(arch_atomic64_add_negative) | |
c020395b | 1575 | static __always_inline bool |
aa525d06 | 1576 | atomic64_add_negative(s64 i, atomic64_t *v) |
b06ed71a | 1577 | { |
ed8af2e4 | 1578 | instrument_atomic_write(v, sizeof(*v)); |
b06ed71a DV |
1579 | return arch_atomic64_add_negative(i, v); |
1580 | } | |
aa525d06 MR |
1581 | #define atomic64_add_negative atomic64_add_negative |
1582 | #endif | |
1583 | ||
1584 | #if defined(arch_atomic64_fetch_add_unless) | |
c020395b | 1585 | static __always_inline s64 |
aa525d06 MR |
1586 | atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) |
1587 | { | |
ed8af2e4 | 1588 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
1589 | return arch_atomic64_fetch_add_unless(v, a, u); |
1590 | } | |
1591 | #define atomic64_fetch_add_unless atomic64_fetch_add_unless | |
1592 | #endif | |
1593 | ||
1594 | #if defined(arch_atomic64_add_unless) | |
c020395b | 1595 | static __always_inline bool |
aa525d06 MR |
1596 | atomic64_add_unless(atomic64_t *v, s64 a, s64 u) |
1597 | { | |
ed8af2e4 | 1598 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
1599 | return arch_atomic64_add_unless(v, a, u); |
1600 | } | |
1601 | #define atomic64_add_unless atomic64_add_unless | |
1602 | #endif | |
1603 | ||
1604 | #if defined(arch_atomic64_inc_not_zero) | |
c020395b | 1605 | static __always_inline bool |
aa525d06 MR |
1606 | atomic64_inc_not_zero(atomic64_t *v) |
1607 | { | |
ed8af2e4 | 1608 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
1609 | return arch_atomic64_inc_not_zero(v); |
1610 | } | |
1611 | #define atomic64_inc_not_zero atomic64_inc_not_zero | |
1612 | #endif | |
1613 | ||
1614 | #if defined(arch_atomic64_inc_unless_negative) | |
c020395b | 1615 | static __always_inline bool |
aa525d06 MR |
1616 | atomic64_inc_unless_negative(atomic64_t *v) |
1617 | { | |
ed8af2e4 | 1618 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
1619 | return arch_atomic64_inc_unless_negative(v); |
1620 | } | |
1621 | #define atomic64_inc_unless_negative atomic64_inc_unless_negative | |
18cc1814 | 1622 | #endif |
b06ed71a | 1623 | |
aa525d06 | 1624 | #if defined(arch_atomic64_dec_unless_positive) |
c020395b | 1625 | static __always_inline bool |
aa525d06 MR |
1626 | atomic64_dec_unless_positive(atomic64_t *v) |
1627 | { | |
ed8af2e4 | 1628 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
1629 | return arch_atomic64_dec_unless_positive(v); |
1630 | } | |
1631 | #define atomic64_dec_unless_positive atomic64_dec_unless_positive | |
1632 | #endif | |
1633 | ||
1634 | #if defined(arch_atomic64_dec_if_positive) | |
c020395b | 1635 | static __always_inline s64 |
aa525d06 MR |
1636 | atomic64_dec_if_positive(atomic64_t *v) |
1637 | { | |
ed8af2e4 | 1638 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
1639 | return arch_atomic64_dec_if_positive(v); |
1640 | } | |
1641 | #define atomic64_dec_if_positive atomic64_dec_if_positive | |
1642 | #endif | |
1643 | ||
1644 | #if !defined(arch_xchg_relaxed) || defined(arch_xchg) | |
1645 | #define xchg(ptr, ...) \ | |
1646 | ({ \ | |
1647 | typeof(ptr) __ai_ptr = (ptr); \ | |
ed8af2e4 | 1648 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 MR |
1649 | arch_xchg(__ai_ptr, __VA_ARGS__); \ |
1650 | }) | |
1651 | #endif | |
1652 | ||
1653 | #if defined(arch_xchg_acquire) | |
1654 | #define xchg_acquire(ptr, ...) \ | |
1655 | ({ \ | |
1656 | typeof(ptr) __ai_ptr = (ptr); \ | |
ed8af2e4 | 1657 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 MR |
1658 | arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ |
1659 | }) | |
1660 | #endif | |
1661 | ||
1662 | #if defined(arch_xchg_release) | |
1663 | #define xchg_release(ptr, ...) \ | |
1664 | ({ \ | |
1665 | typeof(ptr) __ai_ptr = (ptr); \ | |
ed8af2e4 | 1666 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 MR |
1667 | arch_xchg_release(__ai_ptr, __VA_ARGS__); \ |
1668 | }) | |
1669 | #endif | |
1670 | ||
1671 | #if defined(arch_xchg_relaxed) | |
1672 | #define xchg_relaxed(ptr, ...) \ | |
1673 | ({ \ | |
1674 | typeof(ptr) __ai_ptr = (ptr); \ | |
ed8af2e4 | 1675 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 MR |
1676 | arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ |
1677 | }) | |
1678 | #endif | |
1679 | ||
1680 | #if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg) | |
1681 | #define cmpxchg(ptr, ...) \ | |
f9881cc4 MR |
1682 | ({ \ |
1683 | typeof(ptr) __ai_ptr = (ptr); \ | |
ed8af2e4 | 1684 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 | 1685 | arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ |
f9881cc4 | 1686 | }) |
aa525d06 MR |
1687 | #endif |
1688 | ||
1689 | #if defined(arch_cmpxchg_acquire) | |
1690 | #define cmpxchg_acquire(ptr, ...) \ | |
1691 | ({ \ | |
1692 | typeof(ptr) __ai_ptr = (ptr); \ | |
ed8af2e4 | 1693 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 MR |
1694 | arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ |
1695 | }) | |
1696 | #endif | |
1697 | ||
1698 | #if defined(arch_cmpxchg_release) | |
1699 | #define cmpxchg_release(ptr, ...) \ | |
1700 | ({ \ | |
1701 | typeof(ptr) __ai_ptr = (ptr); \ | |
ed8af2e4 | 1702 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 MR |
1703 | arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ |
1704 | }) | |
1705 | #endif | |
1706 | ||
1707 | #if defined(arch_cmpxchg_relaxed) | |
1708 | #define cmpxchg_relaxed(ptr, ...) \ | |
1709 | ({ \ | |
1710 | typeof(ptr) __ai_ptr = (ptr); \ | |
ed8af2e4 | 1711 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 MR |
1712 | arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ |
1713 | }) | |
1714 | #endif | |
f9881cc4 | 1715 | |
aa525d06 MR |
1716 | #if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64) |
1717 | #define cmpxchg64(ptr, ...) \ | |
b06ed71a | 1718 | ({ \ |
df79ed2c | 1719 | typeof(ptr) __ai_ptr = (ptr); \ |
ed8af2e4 | 1720 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 | 1721 | arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ |
b06ed71a | 1722 | }) |
aa525d06 MR |
1723 | #endif |
1724 | ||
1725 | #if defined(arch_cmpxchg64_acquire) | |
1726 | #define cmpxchg64_acquire(ptr, ...) \ | |
1727 | ({ \ | |
1728 | typeof(ptr) __ai_ptr = (ptr); \ | |
ed8af2e4 | 1729 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 MR |
1730 | arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ |
1731 | }) | |
1732 | #endif | |
1733 | ||
1734 | #if defined(arch_cmpxchg64_release) | |
1735 | #define cmpxchg64_release(ptr, ...) \ | |
1736 | ({ \ | |
1737 | typeof(ptr) __ai_ptr = (ptr); \ | |
ed8af2e4 | 1738 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 MR |
1739 | arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ |
1740 | }) | |
1741 | #endif | |
b06ed71a | 1742 | |
aa525d06 MR |
1743 | #if defined(arch_cmpxchg64_relaxed) |
1744 | #define cmpxchg64_relaxed(ptr, ...) \ | |
b06ed71a | 1745 | ({ \ |
df79ed2c | 1746 | typeof(ptr) __ai_ptr = (ptr); \ |
ed8af2e4 | 1747 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 | 1748 | arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ |
b06ed71a | 1749 | }) |
aa525d06 | 1750 | #endif |
b06ed71a | 1751 | |
aa525d06 | 1752 | #define cmpxchg_local(ptr, ...) \ |
b06ed71a | 1753 | ({ \ |
df79ed2c | 1754 | typeof(ptr) __ai_ptr = (ptr); \ |
ed8af2e4 | 1755 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 | 1756 | arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ |
b06ed71a DV |
1757 | }) |
1758 | ||
aa525d06 | 1759 | #define cmpxchg64_local(ptr, ...) \ |
b06ed71a | 1760 | ({ \ |
df79ed2c | 1761 | typeof(ptr) __ai_ptr = (ptr); \ |
ed8af2e4 | 1762 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 | 1763 | arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ |
b06ed71a DV |
1764 | }) |
1765 | ||
aa525d06 | 1766 | #define sync_cmpxchg(ptr, ...) \ |
b06ed71a | 1767 | ({ \ |
df79ed2c | 1768 | typeof(ptr) __ai_ptr = (ptr); \ |
ed8af2e4 | 1769 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
aa525d06 | 1770 | arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ |
b06ed71a DV |
1771 | }) |
1772 | ||
aa525d06 | 1773 | #define cmpxchg_double(ptr, ...) \ |
b06ed71a | 1774 | ({ \ |
aa525d06 | 1775 | typeof(ptr) __ai_ptr = (ptr); \ |
ed8af2e4 | 1776 | instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ |
aa525d06 | 1777 | arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ |
b06ed71a DV |
1778 | }) |
1779 | ||
aa525d06 MR |
1780 | |
1781 | #define cmpxchg_double_local(ptr, ...) \ | |
1782 | ({ \ | |
1783 | typeof(ptr) __ai_ptr = (ptr); \ | |
ed8af2e4 | 1784 | instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ |
aa525d06 | 1785 | arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ |
b06ed71a DV |
1786 | }) |
1787 | ||
aa525d06 | 1788 | #endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */ |
ed8af2e4 | 1789 | // 89bf97f3a7509b740845e51ddf31055b48a81f40 |