Commit | Line | Data |
---|---|---|
aa525d06 MR |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | // Generated by scripts/atomic/gen-atomic-instrumented.sh | |
4 | // DO NOT MODIFY THIS FILE DIRECTLY | |
5 | ||
ac605bee DV |
6 | /* |
7 | * This file provides wrappers with KASAN instrumentation for atomic operations. | |
8 | * To use this functionality an arch's atomic.h file needs to define all | |
9 | * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include | |
10 | * this file at the end. This file provides atomic_read() that forwards to | |
11 | * arch_atomic_read() for actual atomic operation. | |
12 | * Note: if an arch atomic operation is implemented by means of other atomic | |
13 | * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use | |
14 | * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid | |
15 | * double instrumentation. | |
16 | */ | |
aa525d06 MR |
17 | #ifndef _ASM_GENERIC_ATOMIC_INSTRUMENTED_H |
18 | #define _ASM_GENERIC_ATOMIC_INSTRUMENTED_H | |
19 | ||
20 | #include <linux/build_bug.h> | |
c020395b | 21 | #include <linux/compiler.h> |
ed8af2e4 | 22 | #include <linux/instrumented.h> |
aa525d06 | 23 | |
c020395b | 24 | static __always_inline int |
aa525d06 MR |
25 | atomic_read(const atomic_t *v) |
26 | { | |
ed8af2e4 | 27 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
28 | return arch_atomic_read(v); |
29 | } | |
aa525d06 | 30 | |
c020395b | 31 | static __always_inline int |
aa525d06 MR |
32 | atomic_read_acquire(const atomic_t *v) |
33 | { | |
ed8af2e4 | 34 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
35 | return arch_atomic_read_acquire(v); |
36 | } | |
aa525d06 | 37 | |
c020395b | 38 | static __always_inline void |
aa525d06 MR |
39 | atomic_set(atomic_t *v, int i) |
40 | { | |
ed8af2e4 | 41 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
42 | arch_atomic_set(v, i); |
43 | } | |
aa525d06 | 44 | |
c020395b | 45 | static __always_inline void |
aa525d06 MR |
46 | atomic_set_release(atomic_t *v, int i) |
47 | { | |
ed8af2e4 | 48 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
49 | arch_atomic_set_release(v, i); |
50 | } | |
aa525d06 | 51 | |
c020395b | 52 | static __always_inline void |
aa525d06 MR |
53 | atomic_add(int i, atomic_t *v) |
54 | { | |
3570a1bc | 55 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
56 | arch_atomic_add(i, v); |
57 | } | |
aa525d06 | 58 | |
c020395b | 59 | static __always_inline int |
aa525d06 MR |
60 | atomic_add_return(int i, atomic_t *v) |
61 | { | |
3570a1bc | 62 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
63 | return arch_atomic_add_return(i, v); |
64 | } | |
aa525d06 | 65 | |
c020395b | 66 | static __always_inline int |
aa525d06 MR |
67 | atomic_add_return_acquire(int i, atomic_t *v) |
68 | { | |
3570a1bc | 69 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
70 | return arch_atomic_add_return_acquire(i, v); |
71 | } | |
aa525d06 | 72 | |
c020395b | 73 | static __always_inline int |
aa525d06 MR |
74 | atomic_add_return_release(int i, atomic_t *v) |
75 | { | |
3570a1bc | 76 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
77 | return arch_atomic_add_return_release(i, v); |
78 | } | |
aa525d06 | 79 | |
c020395b | 80 | static __always_inline int |
aa525d06 MR |
81 | atomic_add_return_relaxed(int i, atomic_t *v) |
82 | { | |
3570a1bc | 83 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
84 | return arch_atomic_add_return_relaxed(i, v); |
85 | } | |
aa525d06 | 86 | |
c020395b | 87 | static __always_inline int |
aa525d06 MR |
88 | atomic_fetch_add(int i, atomic_t *v) |
89 | { | |
3570a1bc | 90 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
91 | return arch_atomic_fetch_add(i, v); |
92 | } | |
aa525d06 | 93 | |
c020395b | 94 | static __always_inline int |
aa525d06 MR |
95 | atomic_fetch_add_acquire(int i, atomic_t *v) |
96 | { | |
3570a1bc | 97 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
98 | return arch_atomic_fetch_add_acquire(i, v); |
99 | } | |
aa525d06 | 100 | |
c020395b | 101 | static __always_inline int |
aa525d06 MR |
102 | atomic_fetch_add_release(int i, atomic_t *v) |
103 | { | |
3570a1bc | 104 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
105 | return arch_atomic_fetch_add_release(i, v); |
106 | } | |
aa525d06 | 107 | |
c020395b | 108 | static __always_inline int |
aa525d06 MR |
109 | atomic_fetch_add_relaxed(int i, atomic_t *v) |
110 | { | |
3570a1bc | 111 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
112 | return arch_atomic_fetch_add_relaxed(i, v); |
113 | } | |
aa525d06 | 114 | |
c020395b | 115 | static __always_inline void |
aa525d06 MR |
116 | atomic_sub(int i, atomic_t *v) |
117 | { | |
3570a1bc | 118 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
119 | arch_atomic_sub(i, v); |
120 | } | |
aa525d06 | 121 | |
c020395b | 122 | static __always_inline int |
aa525d06 MR |
123 | atomic_sub_return(int i, atomic_t *v) |
124 | { | |
3570a1bc | 125 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
126 | return arch_atomic_sub_return(i, v); |
127 | } | |
aa525d06 | 128 | |
c020395b | 129 | static __always_inline int |
aa525d06 MR |
130 | atomic_sub_return_acquire(int i, atomic_t *v) |
131 | { | |
3570a1bc | 132 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
133 | return arch_atomic_sub_return_acquire(i, v); |
134 | } | |
aa525d06 | 135 | |
c020395b | 136 | static __always_inline int |
aa525d06 MR |
137 | atomic_sub_return_release(int i, atomic_t *v) |
138 | { | |
3570a1bc | 139 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
140 | return arch_atomic_sub_return_release(i, v); |
141 | } | |
aa525d06 | 142 | |
c020395b | 143 | static __always_inline int |
aa525d06 MR |
144 | atomic_sub_return_relaxed(int i, atomic_t *v) |
145 | { | |
3570a1bc | 146 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
147 | return arch_atomic_sub_return_relaxed(i, v); |
148 | } | |
aa525d06 | 149 | |
c020395b | 150 | static __always_inline int |
aa525d06 MR |
151 | atomic_fetch_sub(int i, atomic_t *v) |
152 | { | |
3570a1bc | 153 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
154 | return arch_atomic_fetch_sub(i, v); |
155 | } | |
aa525d06 | 156 | |
c020395b | 157 | static __always_inline int |
aa525d06 MR |
158 | atomic_fetch_sub_acquire(int i, atomic_t *v) |
159 | { | |
3570a1bc | 160 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
161 | return arch_atomic_fetch_sub_acquire(i, v); |
162 | } | |
aa525d06 | 163 | |
c020395b | 164 | static __always_inline int |
aa525d06 MR |
165 | atomic_fetch_sub_release(int i, atomic_t *v) |
166 | { | |
3570a1bc | 167 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
168 | return arch_atomic_fetch_sub_release(i, v); |
169 | } | |
aa525d06 | 170 | |
c020395b | 171 | static __always_inline int |
aa525d06 MR |
172 | atomic_fetch_sub_relaxed(int i, atomic_t *v) |
173 | { | |
3570a1bc | 174 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
175 | return arch_atomic_fetch_sub_relaxed(i, v); |
176 | } | |
aa525d06 | 177 | |
c020395b | 178 | static __always_inline void |
aa525d06 MR |
179 | atomic_inc(atomic_t *v) |
180 | { | |
3570a1bc | 181 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
182 | arch_atomic_inc(v); |
183 | } | |
aa525d06 | 184 | |
c020395b | 185 | static __always_inline int |
aa525d06 MR |
186 | atomic_inc_return(atomic_t *v) |
187 | { | |
3570a1bc | 188 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
189 | return arch_atomic_inc_return(v); |
190 | } | |
aa525d06 | 191 | |
c020395b | 192 | static __always_inline int |
aa525d06 MR |
193 | atomic_inc_return_acquire(atomic_t *v) |
194 | { | |
3570a1bc | 195 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
196 | return arch_atomic_inc_return_acquire(v); |
197 | } | |
aa525d06 | 198 | |
c020395b | 199 | static __always_inline int |
aa525d06 MR |
200 | atomic_inc_return_release(atomic_t *v) |
201 | { | |
3570a1bc | 202 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
203 | return arch_atomic_inc_return_release(v); |
204 | } | |
aa525d06 | 205 | |
c020395b | 206 | static __always_inline int |
aa525d06 MR |
207 | atomic_inc_return_relaxed(atomic_t *v) |
208 | { | |
3570a1bc | 209 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
210 | return arch_atomic_inc_return_relaxed(v); |
211 | } | |
aa525d06 | 212 | |
c020395b | 213 | static __always_inline int |
aa525d06 MR |
214 | atomic_fetch_inc(atomic_t *v) |
215 | { | |
3570a1bc | 216 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
217 | return arch_atomic_fetch_inc(v); |
218 | } | |
aa525d06 | 219 | |
c020395b | 220 | static __always_inline int |
aa525d06 MR |
221 | atomic_fetch_inc_acquire(atomic_t *v) |
222 | { | |
3570a1bc | 223 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
224 | return arch_atomic_fetch_inc_acquire(v); |
225 | } | |
aa525d06 | 226 | |
c020395b | 227 | static __always_inline int |
aa525d06 MR |
228 | atomic_fetch_inc_release(atomic_t *v) |
229 | { | |
3570a1bc | 230 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
231 | return arch_atomic_fetch_inc_release(v); |
232 | } | |
aa525d06 | 233 | |
c020395b | 234 | static __always_inline int |
aa525d06 MR |
235 | atomic_fetch_inc_relaxed(atomic_t *v) |
236 | { | |
3570a1bc | 237 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
238 | return arch_atomic_fetch_inc_relaxed(v); |
239 | } | |
aa525d06 | 240 | |
c020395b | 241 | static __always_inline void |
aa525d06 MR |
242 | atomic_dec(atomic_t *v) |
243 | { | |
3570a1bc | 244 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
245 | arch_atomic_dec(v); |
246 | } | |
aa525d06 | 247 | |
c020395b | 248 | static __always_inline int |
aa525d06 MR |
249 | atomic_dec_return(atomic_t *v) |
250 | { | |
3570a1bc | 251 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
252 | return arch_atomic_dec_return(v); |
253 | } | |
aa525d06 | 254 | |
c020395b | 255 | static __always_inline int |
aa525d06 MR |
256 | atomic_dec_return_acquire(atomic_t *v) |
257 | { | |
3570a1bc | 258 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
259 | return arch_atomic_dec_return_acquire(v); |
260 | } | |
aa525d06 | 261 | |
c020395b | 262 | static __always_inline int |
aa525d06 MR |
263 | atomic_dec_return_release(atomic_t *v) |
264 | { | |
3570a1bc | 265 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
266 | return arch_atomic_dec_return_release(v); |
267 | } | |
aa525d06 | 268 | |
c020395b | 269 | static __always_inline int |
aa525d06 MR |
270 | atomic_dec_return_relaxed(atomic_t *v) |
271 | { | |
3570a1bc | 272 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
273 | return arch_atomic_dec_return_relaxed(v); |
274 | } | |
aa525d06 | 275 | |
c020395b | 276 | static __always_inline int |
aa525d06 MR |
277 | atomic_fetch_dec(atomic_t *v) |
278 | { | |
3570a1bc | 279 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
280 | return arch_atomic_fetch_dec(v); |
281 | } | |
aa525d06 | 282 | |
c020395b | 283 | static __always_inline int |
aa525d06 MR |
284 | atomic_fetch_dec_acquire(atomic_t *v) |
285 | { | |
3570a1bc | 286 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
287 | return arch_atomic_fetch_dec_acquire(v); |
288 | } | |
aa525d06 | 289 | |
c020395b | 290 | static __always_inline int |
aa525d06 MR |
291 | atomic_fetch_dec_release(atomic_t *v) |
292 | { | |
3570a1bc | 293 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
294 | return arch_atomic_fetch_dec_release(v); |
295 | } | |
aa525d06 | 296 | |
c020395b | 297 | static __always_inline int |
aa525d06 MR |
298 | atomic_fetch_dec_relaxed(atomic_t *v) |
299 | { | |
3570a1bc | 300 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
301 | return arch_atomic_fetch_dec_relaxed(v); |
302 | } | |
aa525d06 | 303 | |
c020395b | 304 | static __always_inline void |
aa525d06 MR |
305 | atomic_and(int i, atomic_t *v) |
306 | { | |
3570a1bc | 307 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
308 | arch_atomic_and(i, v); |
309 | } | |
aa525d06 | 310 | |
c020395b | 311 | static __always_inline int |
aa525d06 MR |
312 | atomic_fetch_and(int i, atomic_t *v) |
313 | { | |
3570a1bc | 314 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
315 | return arch_atomic_fetch_and(i, v); |
316 | } | |
aa525d06 | 317 | |
c020395b | 318 | static __always_inline int |
aa525d06 MR |
319 | atomic_fetch_and_acquire(int i, atomic_t *v) |
320 | { | |
3570a1bc | 321 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
322 | return arch_atomic_fetch_and_acquire(i, v); |
323 | } | |
aa525d06 | 324 | |
c020395b | 325 | static __always_inline int |
aa525d06 MR |
326 | atomic_fetch_and_release(int i, atomic_t *v) |
327 | { | |
3570a1bc | 328 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
329 | return arch_atomic_fetch_and_release(i, v); |
330 | } | |
aa525d06 | 331 | |
c020395b | 332 | static __always_inline int |
aa525d06 MR |
333 | atomic_fetch_and_relaxed(int i, atomic_t *v) |
334 | { | |
3570a1bc | 335 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
336 | return arch_atomic_fetch_and_relaxed(i, v); |
337 | } | |
aa525d06 | 338 | |
c020395b | 339 | static __always_inline void |
aa525d06 MR |
340 | atomic_andnot(int i, atomic_t *v) |
341 | { | |
3570a1bc | 342 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
343 | arch_atomic_andnot(i, v); |
344 | } | |
aa525d06 | 345 | |
c020395b | 346 | static __always_inline int |
aa525d06 MR |
347 | atomic_fetch_andnot(int i, atomic_t *v) |
348 | { | |
3570a1bc | 349 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
350 | return arch_atomic_fetch_andnot(i, v); |
351 | } | |
aa525d06 | 352 | |
c020395b | 353 | static __always_inline int |
aa525d06 MR |
354 | atomic_fetch_andnot_acquire(int i, atomic_t *v) |
355 | { | |
3570a1bc | 356 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
357 | return arch_atomic_fetch_andnot_acquire(i, v); |
358 | } | |
aa525d06 | 359 | |
c020395b | 360 | static __always_inline int |
aa525d06 MR |
361 | atomic_fetch_andnot_release(int i, atomic_t *v) |
362 | { | |
3570a1bc | 363 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
364 | return arch_atomic_fetch_andnot_release(i, v); |
365 | } | |
aa525d06 | 366 | |
c020395b | 367 | static __always_inline int |
aa525d06 MR |
368 | atomic_fetch_andnot_relaxed(int i, atomic_t *v) |
369 | { | |
3570a1bc | 370 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
371 | return arch_atomic_fetch_andnot_relaxed(i, v); |
372 | } | |
aa525d06 | 373 | |
c020395b | 374 | static __always_inline void |
aa525d06 MR |
375 | atomic_or(int i, atomic_t *v) |
376 | { | |
3570a1bc | 377 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
378 | arch_atomic_or(i, v); |
379 | } | |
aa525d06 | 380 | |
c020395b | 381 | static __always_inline int |
aa525d06 MR |
382 | atomic_fetch_or(int i, atomic_t *v) |
383 | { | |
3570a1bc | 384 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
385 | return arch_atomic_fetch_or(i, v); |
386 | } | |
aa525d06 | 387 | |
c020395b | 388 | static __always_inline int |
aa525d06 MR |
389 | atomic_fetch_or_acquire(int i, atomic_t *v) |
390 | { | |
3570a1bc | 391 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
392 | return arch_atomic_fetch_or_acquire(i, v); |
393 | } | |
aa525d06 | 394 | |
c020395b | 395 | static __always_inline int |
aa525d06 MR |
396 | atomic_fetch_or_release(int i, atomic_t *v) |
397 | { | |
3570a1bc | 398 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
399 | return arch_atomic_fetch_or_release(i, v); |
400 | } | |
aa525d06 | 401 | |
c020395b | 402 | static __always_inline int |
aa525d06 MR |
403 | atomic_fetch_or_relaxed(int i, atomic_t *v) |
404 | { | |
3570a1bc | 405 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
406 | return arch_atomic_fetch_or_relaxed(i, v); |
407 | } | |
aa525d06 | 408 | |
c020395b | 409 | static __always_inline void |
aa525d06 MR |
410 | atomic_xor(int i, atomic_t *v) |
411 | { | |
3570a1bc | 412 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
413 | arch_atomic_xor(i, v); |
414 | } | |
aa525d06 | 415 | |
c020395b | 416 | static __always_inline int |
aa525d06 MR |
417 | atomic_fetch_xor(int i, atomic_t *v) |
418 | { | |
3570a1bc | 419 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
420 | return arch_atomic_fetch_xor(i, v); |
421 | } | |
aa525d06 | 422 | |
c020395b | 423 | static __always_inline int |
aa525d06 MR |
424 | atomic_fetch_xor_acquire(int i, atomic_t *v) |
425 | { | |
3570a1bc | 426 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
427 | return arch_atomic_fetch_xor_acquire(i, v); |
428 | } | |
aa525d06 | 429 | |
c020395b | 430 | static __always_inline int |
aa525d06 MR |
431 | atomic_fetch_xor_release(int i, atomic_t *v) |
432 | { | |
3570a1bc | 433 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
434 | return arch_atomic_fetch_xor_release(i, v); |
435 | } | |
aa525d06 | 436 | |
c020395b | 437 | static __always_inline int |
aa525d06 MR |
438 | atomic_fetch_xor_relaxed(int i, atomic_t *v) |
439 | { | |
3570a1bc | 440 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
441 | return arch_atomic_fetch_xor_relaxed(i, v); |
442 | } | |
aa525d06 | 443 | |
c020395b | 444 | static __always_inline int |
aa525d06 MR |
445 | atomic_xchg(atomic_t *v, int i) |
446 | { | |
3570a1bc | 447 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
448 | return arch_atomic_xchg(v, i); |
449 | } | |
aa525d06 | 450 | |
c020395b | 451 | static __always_inline int |
aa525d06 MR |
452 | atomic_xchg_acquire(atomic_t *v, int i) |
453 | { | |
3570a1bc | 454 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
455 | return arch_atomic_xchg_acquire(v, i); |
456 | } | |
aa525d06 | 457 | |
c020395b | 458 | static __always_inline int |
aa525d06 MR |
459 | atomic_xchg_release(atomic_t *v, int i) |
460 | { | |
3570a1bc | 461 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
462 | return arch_atomic_xchg_release(v, i); |
463 | } | |
aa525d06 | 464 | |
c020395b | 465 | static __always_inline int |
aa525d06 MR |
466 | atomic_xchg_relaxed(atomic_t *v, int i) |
467 | { | |
3570a1bc | 468 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
469 | return arch_atomic_xchg_relaxed(v, i); |
470 | } | |
aa525d06 | 471 | |
c020395b | 472 | static __always_inline int |
aa525d06 MR |
473 | atomic_cmpxchg(atomic_t *v, int old, int new) |
474 | { | |
3570a1bc | 475 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
476 | return arch_atomic_cmpxchg(v, old, new); |
477 | } | |
aa525d06 | 478 | |
c020395b | 479 | static __always_inline int |
aa525d06 MR |
480 | atomic_cmpxchg_acquire(atomic_t *v, int old, int new) |
481 | { | |
3570a1bc | 482 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
483 | return arch_atomic_cmpxchg_acquire(v, old, new); |
484 | } | |
aa525d06 | 485 | |
c020395b | 486 | static __always_inline int |
aa525d06 MR |
487 | atomic_cmpxchg_release(atomic_t *v, int old, int new) |
488 | { | |
3570a1bc | 489 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
490 | return arch_atomic_cmpxchg_release(v, old, new); |
491 | } | |
aa525d06 | 492 | |
c020395b | 493 | static __always_inline int |
aa525d06 MR |
494 | atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) |
495 | { | |
3570a1bc | 496 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
497 | return arch_atomic_cmpxchg_relaxed(v, old, new); |
498 | } | |
aa525d06 | 499 | |
c020395b | 500 | static __always_inline bool |
aa525d06 MR |
501 | atomic_try_cmpxchg(atomic_t *v, int *old, int new) |
502 | { | |
3570a1bc ME |
503 | instrument_atomic_read_write(v, sizeof(*v)); |
504 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 MR |
505 | return arch_atomic_try_cmpxchg(v, old, new); |
506 | } | |
aa525d06 | 507 | |
c020395b | 508 | static __always_inline bool |
aa525d06 MR |
509 | atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) |
510 | { | |
3570a1bc ME |
511 | instrument_atomic_read_write(v, sizeof(*v)); |
512 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 MR |
513 | return arch_atomic_try_cmpxchg_acquire(v, old, new); |
514 | } | |
aa525d06 | 515 | |
c020395b | 516 | static __always_inline bool |
aa525d06 MR |
517 | atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) |
518 | { | |
3570a1bc ME |
519 | instrument_atomic_read_write(v, sizeof(*v)); |
520 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 MR |
521 | return arch_atomic_try_cmpxchg_release(v, old, new); |
522 | } | |
aa525d06 | 523 | |
c020395b | 524 | static __always_inline bool |
aa525d06 MR |
525 | atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) |
526 | { | |
3570a1bc ME |
527 | instrument_atomic_read_write(v, sizeof(*v)); |
528 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 MR |
529 | return arch_atomic_try_cmpxchg_relaxed(v, old, new); |
530 | } | |
aa525d06 | 531 | |
c020395b | 532 | static __always_inline bool |
aa525d06 MR |
533 | atomic_sub_and_test(int i, atomic_t *v) |
534 | { | |
3570a1bc | 535 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
536 | return arch_atomic_sub_and_test(i, v); |
537 | } | |
aa525d06 | 538 | |
c020395b | 539 | static __always_inline bool |
aa525d06 MR |
540 | atomic_dec_and_test(atomic_t *v) |
541 | { | |
3570a1bc | 542 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
543 | return arch_atomic_dec_and_test(v); |
544 | } | |
aa525d06 | 545 | |
c020395b | 546 | static __always_inline bool |
aa525d06 MR |
547 | atomic_inc_and_test(atomic_t *v) |
548 | { | |
3570a1bc | 549 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
550 | return arch_atomic_inc_and_test(v); |
551 | } | |
aa525d06 | 552 | |
c020395b | 553 | static __always_inline bool |
aa525d06 MR |
554 | atomic_add_negative(int i, atomic_t *v) |
555 | { | |
3570a1bc | 556 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
557 | return arch_atomic_add_negative(i, v); |
558 | } | |
aa525d06 | 559 | |
c020395b | 560 | static __always_inline int |
aa525d06 MR |
561 | atomic_fetch_add_unless(atomic_t *v, int a, int u) |
562 | { | |
3570a1bc | 563 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
564 | return arch_atomic_fetch_add_unless(v, a, u); |
565 | } | |
aa525d06 | 566 | |
c020395b | 567 | static __always_inline bool |
aa525d06 MR |
568 | atomic_add_unless(atomic_t *v, int a, int u) |
569 | { | |
3570a1bc | 570 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
571 | return arch_atomic_add_unless(v, a, u); |
572 | } | |
aa525d06 | 573 | |
c020395b | 574 | static __always_inline bool |
aa525d06 MR |
575 | atomic_inc_not_zero(atomic_t *v) |
576 | { | |
3570a1bc | 577 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
578 | return arch_atomic_inc_not_zero(v); |
579 | } | |
aa525d06 | 580 | |
c020395b | 581 | static __always_inline bool |
aa525d06 MR |
582 | atomic_inc_unless_negative(atomic_t *v) |
583 | { | |
3570a1bc | 584 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
585 | return arch_atomic_inc_unless_negative(v); |
586 | } | |
aa525d06 | 587 | |
c020395b | 588 | static __always_inline bool |
aa525d06 MR |
589 | atomic_dec_unless_positive(atomic_t *v) |
590 | { | |
3570a1bc | 591 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
592 | return arch_atomic_dec_unless_positive(v); |
593 | } | |
aa525d06 | 594 | |
c020395b | 595 | static __always_inline int |
aa525d06 MR |
596 | atomic_dec_if_positive(atomic_t *v) |
597 | { | |
3570a1bc | 598 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
599 | return arch_atomic_dec_if_positive(v); |
600 | } | |
aa525d06 | 601 | |
c020395b | 602 | static __always_inline s64 |
aa525d06 MR |
603 | atomic64_read(const atomic64_t *v) |
604 | { | |
ed8af2e4 | 605 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
606 | return arch_atomic64_read(v); |
607 | } | |
aa525d06 | 608 | |
c020395b | 609 | static __always_inline s64 |
aa525d06 MR |
610 | atomic64_read_acquire(const atomic64_t *v) |
611 | { | |
ed8af2e4 | 612 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
613 | return arch_atomic64_read_acquire(v); |
614 | } | |
aa525d06 | 615 | |
c020395b | 616 | static __always_inline void |
aa525d06 MR |
617 | atomic64_set(atomic64_t *v, s64 i) |
618 | { | |
ed8af2e4 | 619 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
620 | arch_atomic64_set(v, i); |
621 | } | |
aa525d06 | 622 | |
c020395b | 623 | static __always_inline void |
aa525d06 MR |
624 | atomic64_set_release(atomic64_t *v, s64 i) |
625 | { | |
ed8af2e4 | 626 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
627 | arch_atomic64_set_release(v, i); |
628 | } | |
aa525d06 | 629 | |
c020395b | 630 | static __always_inline void |
aa525d06 MR |
631 | atomic64_add(s64 i, atomic64_t *v) |
632 | { | |
3570a1bc | 633 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
634 | arch_atomic64_add(i, v); |
635 | } | |
aa525d06 | 636 | |
c020395b | 637 | static __always_inline s64 |
aa525d06 MR |
638 | atomic64_add_return(s64 i, atomic64_t *v) |
639 | { | |
3570a1bc | 640 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
641 | return arch_atomic64_add_return(i, v); |
642 | } | |
aa525d06 | 643 | |
c020395b | 644 | static __always_inline s64 |
aa525d06 MR |
645 | atomic64_add_return_acquire(s64 i, atomic64_t *v) |
646 | { | |
3570a1bc | 647 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
648 | return arch_atomic64_add_return_acquire(i, v); |
649 | } | |
aa525d06 | 650 | |
c020395b | 651 | static __always_inline s64 |
aa525d06 MR |
652 | atomic64_add_return_release(s64 i, atomic64_t *v) |
653 | { | |
3570a1bc | 654 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
655 | return arch_atomic64_add_return_release(i, v); |
656 | } | |
aa525d06 | 657 | |
c020395b | 658 | static __always_inline s64 |
aa525d06 MR |
659 | atomic64_add_return_relaxed(s64 i, atomic64_t *v) |
660 | { | |
3570a1bc | 661 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
662 | return arch_atomic64_add_return_relaxed(i, v); |
663 | } | |
aa525d06 | 664 | |
c020395b | 665 | static __always_inline s64 |
aa525d06 MR |
666 | atomic64_fetch_add(s64 i, atomic64_t *v) |
667 | { | |
3570a1bc | 668 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
669 | return arch_atomic64_fetch_add(i, v); |
670 | } | |
aa525d06 | 671 | |
c020395b | 672 | static __always_inline s64 |
aa525d06 MR |
673 | atomic64_fetch_add_acquire(s64 i, atomic64_t *v) |
674 | { | |
3570a1bc | 675 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
676 | return arch_atomic64_fetch_add_acquire(i, v); |
677 | } | |
aa525d06 | 678 | |
c020395b | 679 | static __always_inline s64 |
aa525d06 MR |
680 | atomic64_fetch_add_release(s64 i, atomic64_t *v) |
681 | { | |
3570a1bc | 682 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
683 | return arch_atomic64_fetch_add_release(i, v); |
684 | } | |
aa525d06 | 685 | |
c020395b | 686 | static __always_inline s64 |
aa525d06 MR |
687 | atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) |
688 | { | |
3570a1bc | 689 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
690 | return arch_atomic64_fetch_add_relaxed(i, v); |
691 | } | |
aa525d06 | 692 | |
c020395b | 693 | static __always_inline void |
aa525d06 MR |
694 | atomic64_sub(s64 i, atomic64_t *v) |
695 | { | |
3570a1bc | 696 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
697 | arch_atomic64_sub(i, v); |
698 | } | |
ac605bee | 699 | |
c020395b | 700 | static __always_inline s64 |
aa525d06 MR |
701 | atomic64_sub_return(s64 i, atomic64_t *v) |
702 | { | |
3570a1bc | 703 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
704 | return arch_atomic64_sub_return(i, v); |
705 | } | |
b06ed71a | 706 | |
c020395b | 707 | static __always_inline s64 |
aa525d06 MR |
708 | atomic64_sub_return_acquire(s64 i, atomic64_t *v) |
709 | { | |
3570a1bc | 710 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
711 | return arch_atomic64_sub_return_acquire(i, v); |
712 | } | |
b06ed71a | 713 | |
c020395b | 714 | static __always_inline s64 |
aa525d06 | 715 | atomic64_sub_return_release(s64 i, atomic64_t *v) |
b06ed71a | 716 | { |
3570a1bc | 717 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 718 | return arch_atomic64_sub_return_release(i, v); |
b06ed71a DV |
719 | } |
720 | ||
c020395b | 721 | static __always_inline s64 |
aa525d06 | 722 | atomic64_sub_return_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 723 | { |
3570a1bc | 724 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 725 | return arch_atomic64_sub_return_relaxed(i, v); |
b06ed71a DV |
726 | } |
727 | ||
c020395b | 728 | static __always_inline s64 |
aa525d06 | 729 | atomic64_fetch_sub(s64 i, atomic64_t *v) |
b06ed71a | 730 | { |
3570a1bc | 731 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 732 | return arch_atomic64_fetch_sub(i, v); |
b06ed71a DV |
733 | } |
734 | ||
c020395b | 735 | static __always_inline s64 |
aa525d06 | 736 | atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) |
b06ed71a | 737 | { |
3570a1bc | 738 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 739 | return arch_atomic64_fetch_sub_acquire(i, v); |
b06ed71a DV |
740 | } |
741 | ||
c020395b | 742 | static __always_inline s64 |
aa525d06 | 743 | atomic64_fetch_sub_release(s64 i, atomic64_t *v) |
b06ed71a | 744 | { |
3570a1bc | 745 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 746 | return arch_atomic64_fetch_sub_release(i, v); |
b06ed71a DV |
747 | } |
748 | ||
c020395b | 749 | static __always_inline s64 |
aa525d06 | 750 | atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 751 | { |
3570a1bc | 752 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 753 | return arch_atomic64_fetch_sub_relaxed(i, v); |
b06ed71a DV |
754 | } |
755 | ||
c020395b | 756 | static __always_inline void |
aa525d06 | 757 | atomic64_inc(atomic64_t *v) |
b06ed71a | 758 | { |
3570a1bc | 759 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 760 | arch_atomic64_inc(v); |
b06ed71a DV |
761 | } |
762 | ||
c020395b | 763 | static __always_inline s64 |
aa525d06 | 764 | atomic64_inc_return(atomic64_t *v) |
b06ed71a | 765 | { |
3570a1bc | 766 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 767 | return arch_atomic64_inc_return(v); |
b06ed71a DV |
768 | } |
769 | ||
c020395b | 770 | static __always_inline s64 |
aa525d06 | 771 | atomic64_inc_return_acquire(atomic64_t *v) |
b06ed71a | 772 | { |
3570a1bc | 773 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 774 | return arch_atomic64_inc_return_acquire(v); |
b06ed71a | 775 | } |
b06ed71a | 776 | |
c020395b | 777 | static __always_inline s64 |
aa525d06 | 778 | atomic64_inc_return_release(atomic64_t *v) |
b06ed71a | 779 | { |
3570a1bc | 780 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 781 | return arch_atomic64_inc_return_release(v); |
b06ed71a | 782 | } |
b06ed71a | 783 | |
c020395b | 784 | static __always_inline s64 |
aa525d06 | 785 | atomic64_inc_return_relaxed(atomic64_t *v) |
b06ed71a | 786 | { |
3570a1bc | 787 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 788 | return arch_atomic64_inc_return_relaxed(v); |
b06ed71a | 789 | } |
b06ed71a | 790 | |
c020395b | 791 | static __always_inline s64 |
aa525d06 | 792 | atomic64_fetch_inc(atomic64_t *v) |
0ae1d994 | 793 | { |
3570a1bc | 794 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 795 | return arch_atomic64_fetch_inc(v); |
0ae1d994 | 796 | } |
b06ed71a | 797 | |
c020395b | 798 | static __always_inline s64 |
aa525d06 | 799 | atomic64_fetch_inc_acquire(atomic64_t *v) |
b06ed71a | 800 | { |
3570a1bc | 801 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 802 | return arch_atomic64_fetch_inc_acquire(v); |
b06ed71a DV |
803 | } |
804 | ||
c020395b | 805 | static __always_inline s64 |
aa525d06 | 806 | atomic64_fetch_inc_release(atomic64_t *v) |
b06ed71a | 807 | { |
3570a1bc | 808 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 809 | return arch_atomic64_fetch_inc_release(v); |
b06ed71a DV |
810 | } |
811 | ||
c020395b | 812 | static __always_inline s64 |
aa525d06 | 813 | atomic64_fetch_inc_relaxed(atomic64_t *v) |
b06ed71a | 814 | { |
3570a1bc | 815 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 816 | return arch_atomic64_fetch_inc_relaxed(v); |
b06ed71a DV |
817 | } |
818 | ||
c020395b | 819 | static __always_inline void |
aa525d06 | 820 | atomic64_dec(atomic64_t *v) |
b06ed71a | 821 | { |
3570a1bc | 822 | instrument_atomic_read_write(v, sizeof(*v)); |
b06ed71a DV |
823 | arch_atomic64_dec(v); |
824 | } | |
825 | ||
c020395b | 826 | static __always_inline s64 |
aa525d06 | 827 | atomic64_dec_return(atomic64_t *v) |
b06ed71a | 828 | { |
3570a1bc | 829 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 830 | return arch_atomic64_dec_return(v); |
b06ed71a DV |
831 | } |
832 | ||
c020395b | 833 | static __always_inline s64 |
aa525d06 | 834 | atomic64_dec_return_acquire(atomic64_t *v) |
b06ed71a | 835 | { |
3570a1bc | 836 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 837 | return arch_atomic64_dec_return_acquire(v); |
b06ed71a DV |
838 | } |
839 | ||
c020395b | 840 | static __always_inline s64 |
aa525d06 | 841 | atomic64_dec_return_release(atomic64_t *v) |
b06ed71a | 842 | { |
3570a1bc | 843 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 844 | return arch_atomic64_dec_return_release(v); |
b06ed71a DV |
845 | } |
846 | ||
c020395b | 847 | static __always_inline s64 |
aa525d06 | 848 | atomic64_dec_return_relaxed(atomic64_t *v) |
b06ed71a | 849 | { |
3570a1bc | 850 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 851 | return arch_atomic64_dec_return_relaxed(v); |
b06ed71a DV |
852 | } |
853 | ||
c020395b | 854 | static __always_inline s64 |
aa525d06 | 855 | atomic64_fetch_dec(atomic64_t *v) |
b06ed71a | 856 | { |
3570a1bc | 857 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
858 | return arch_atomic64_fetch_dec(v); |
859 | } | |
aa525d06 | 860 | |
c020395b | 861 | static __always_inline s64 |
aa525d06 MR |
862 | atomic64_fetch_dec_acquire(atomic64_t *v) |
863 | { | |
3570a1bc | 864 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
865 | return arch_atomic64_fetch_dec_acquire(v); |
866 | } | |
aa525d06 | 867 | |
c020395b | 868 | static __always_inline s64 |
aa525d06 MR |
869 | atomic64_fetch_dec_release(atomic64_t *v) |
870 | { | |
3570a1bc | 871 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 872 | return arch_atomic64_fetch_dec_release(v); |
b06ed71a | 873 | } |
aa525d06 | 874 | |
c020395b | 875 | static __always_inline s64 |
aa525d06 MR |
876 | atomic64_fetch_dec_relaxed(atomic64_t *v) |
877 | { | |
3570a1bc | 878 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
879 | return arch_atomic64_fetch_dec_relaxed(v); |
880 | } | |
b06ed71a | 881 | |
c020395b | 882 | static __always_inline void |
aa525d06 | 883 | atomic64_and(s64 i, atomic64_t *v) |
b06ed71a | 884 | { |
3570a1bc | 885 | instrument_atomic_read_write(v, sizeof(*v)); |
b06ed71a DV |
886 | arch_atomic64_and(i, v); |
887 | } | |
888 | ||
c020395b | 889 | static __always_inline s64 |
aa525d06 | 890 | atomic64_fetch_and(s64 i, atomic64_t *v) |
b06ed71a | 891 | { |
3570a1bc | 892 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 893 | return arch_atomic64_fetch_and(i, v); |
b06ed71a DV |
894 | } |
895 | ||
c020395b | 896 | static __always_inline s64 |
aa525d06 | 897 | atomic64_fetch_and_acquire(s64 i, atomic64_t *v) |
b06ed71a | 898 | { |
3570a1bc | 899 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 900 | return arch_atomic64_fetch_and_acquire(i, v); |
b06ed71a DV |
901 | } |
902 | ||
c020395b | 903 | static __always_inline s64 |
aa525d06 | 904 | atomic64_fetch_and_release(s64 i, atomic64_t *v) |
b06ed71a | 905 | { |
3570a1bc | 906 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 907 | return arch_atomic64_fetch_and_release(i, v); |
b06ed71a DV |
908 | } |
909 | ||
c020395b | 910 | static __always_inline s64 |
aa525d06 | 911 | atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 912 | { |
3570a1bc | 913 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 914 | return arch_atomic64_fetch_and_relaxed(i, v); |
b06ed71a DV |
915 | } |
916 | ||
c020395b | 917 | static __always_inline void |
aa525d06 | 918 | atomic64_andnot(s64 i, atomic64_t *v) |
b06ed71a | 919 | { |
3570a1bc | 920 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 921 | arch_atomic64_andnot(i, v); |
b06ed71a DV |
922 | } |
923 | ||
c020395b | 924 | static __always_inline s64 |
aa525d06 | 925 | atomic64_fetch_andnot(s64 i, atomic64_t *v) |
b06ed71a | 926 | { |
3570a1bc | 927 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 928 | return arch_atomic64_fetch_andnot(i, v); |
b06ed71a DV |
929 | } |
930 | ||
c020395b | 931 | static __always_inline s64 |
aa525d06 | 932 | atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) |
b06ed71a | 933 | { |
3570a1bc | 934 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 935 | return arch_atomic64_fetch_andnot_acquire(i, v); |
b06ed71a DV |
936 | } |
937 | ||
c020395b | 938 | static __always_inline s64 |
aa525d06 | 939 | atomic64_fetch_andnot_release(s64 i, atomic64_t *v) |
b06ed71a | 940 | { |
3570a1bc | 941 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 942 | return arch_atomic64_fetch_andnot_release(i, v); |
b06ed71a DV |
943 | } |
944 | ||
c020395b | 945 | static __always_inline s64 |
aa525d06 | 946 | atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 947 | { |
3570a1bc | 948 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 949 | return arch_atomic64_fetch_andnot_relaxed(i, v); |
b06ed71a DV |
950 | } |
951 | ||
c020395b | 952 | static __always_inline void |
aa525d06 | 953 | atomic64_or(s64 i, atomic64_t *v) |
b06ed71a | 954 | { |
3570a1bc | 955 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
956 | arch_atomic64_or(i, v); |
957 | } | |
aa525d06 | 958 | |
c020395b | 959 | static __always_inline s64 |
aa525d06 MR |
960 | atomic64_fetch_or(s64 i, atomic64_t *v) |
961 | { | |
3570a1bc | 962 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 963 | return arch_atomic64_fetch_or(i, v); |
b06ed71a DV |
964 | } |
965 | ||
c020395b | 966 | static __always_inline s64 |
aa525d06 | 967 | atomic64_fetch_or_acquire(s64 i, atomic64_t *v) |
b06ed71a | 968 | { |
3570a1bc | 969 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 970 | return arch_atomic64_fetch_or_acquire(i, v); |
b06ed71a DV |
971 | } |
972 | ||
c020395b | 973 | static __always_inline s64 |
aa525d06 | 974 | atomic64_fetch_or_release(s64 i, atomic64_t *v) |
b06ed71a | 975 | { |
3570a1bc | 976 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 977 | return arch_atomic64_fetch_or_release(i, v); |
b06ed71a DV |
978 | } |
979 | ||
c020395b | 980 | static __always_inline s64 |
aa525d06 | 981 | atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 982 | { |
3570a1bc | 983 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 984 | return arch_atomic64_fetch_or_relaxed(i, v); |
b06ed71a DV |
985 | } |
986 | ||
c020395b | 987 | static __always_inline void |
aa525d06 | 988 | atomic64_xor(s64 i, atomic64_t *v) |
b06ed71a | 989 | { |
3570a1bc | 990 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
991 | arch_atomic64_xor(i, v); |
992 | } | |
aa525d06 | 993 | |
c020395b | 994 | static __always_inline s64 |
aa525d06 MR |
995 | atomic64_fetch_xor(s64 i, atomic64_t *v) |
996 | { | |
3570a1bc | 997 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 998 | return arch_atomic64_fetch_xor(i, v); |
b06ed71a DV |
999 | } |
1000 | ||
c020395b | 1001 | static __always_inline s64 |
aa525d06 | 1002 | atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) |
b06ed71a | 1003 | { |
3570a1bc | 1004 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1005 | return arch_atomic64_fetch_xor_acquire(i, v); |
b06ed71a DV |
1006 | } |
1007 | ||
c020395b | 1008 | static __always_inline s64 |
aa525d06 | 1009 | atomic64_fetch_xor_release(s64 i, atomic64_t *v) |
b06ed71a | 1010 | { |
3570a1bc | 1011 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1012 | return arch_atomic64_fetch_xor_release(i, v); |
b06ed71a DV |
1013 | } |
1014 | ||
c020395b | 1015 | static __always_inline s64 |
aa525d06 | 1016 | atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 1017 | { |
3570a1bc | 1018 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1019 | return arch_atomic64_fetch_xor_relaxed(i, v); |
b06ed71a DV |
1020 | } |
1021 | ||
c020395b | 1022 | static __always_inline s64 |
aa525d06 | 1023 | atomic64_xchg(atomic64_t *v, s64 i) |
b06ed71a | 1024 | { |
3570a1bc | 1025 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1026 | return arch_atomic64_xchg(v, i); |
b06ed71a DV |
1027 | } |
1028 | ||
c020395b | 1029 | static __always_inline s64 |
aa525d06 | 1030 | atomic64_xchg_acquire(atomic64_t *v, s64 i) |
b06ed71a | 1031 | { |
3570a1bc | 1032 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1033 | return arch_atomic64_xchg_acquire(v, i); |
b06ed71a DV |
1034 | } |
1035 | ||
c020395b | 1036 | static __always_inline s64 |
aa525d06 | 1037 | atomic64_xchg_release(atomic64_t *v, s64 i) |
b06ed71a | 1038 | { |
3570a1bc | 1039 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1040 | return arch_atomic64_xchg_release(v, i); |
b06ed71a DV |
1041 | } |
1042 | ||
c020395b | 1043 | static __always_inline s64 |
aa525d06 | 1044 | atomic64_xchg_relaxed(atomic64_t *v, s64 i) |
b06ed71a | 1045 | { |
3570a1bc | 1046 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1047 | return arch_atomic64_xchg_relaxed(v, i); |
b06ed71a DV |
1048 | } |
1049 | ||
c020395b | 1050 | static __always_inline s64 |
aa525d06 | 1051 | atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1052 | { |
3570a1bc | 1053 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1054 | return arch_atomic64_cmpxchg(v, old, new); |
b06ed71a DV |
1055 | } |
1056 | ||
c020395b | 1057 | static __always_inline s64 |
aa525d06 | 1058 | atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1059 | { |
3570a1bc | 1060 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1061 | return arch_atomic64_cmpxchg_acquire(v, old, new); |
b06ed71a DV |
1062 | } |
1063 | ||
c020395b | 1064 | static __always_inline s64 |
aa525d06 | 1065 | atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1066 | { |
3570a1bc | 1067 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1068 | return arch_atomic64_cmpxchg_release(v, old, new); |
b06ed71a DV |
1069 | } |
1070 | ||
c020395b | 1071 | static __always_inline s64 |
aa525d06 | 1072 | atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1073 | { |
3570a1bc | 1074 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1075 | return arch_atomic64_cmpxchg_relaxed(v, old, new); |
b06ed71a DV |
1076 | } |
1077 | ||
c020395b | 1078 | static __always_inline bool |
aa525d06 | 1079 | atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1080 | { |
3570a1bc ME |
1081 | instrument_atomic_read_write(v, sizeof(*v)); |
1082 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 | 1083 | return arch_atomic64_try_cmpxchg(v, old, new); |
b06ed71a DV |
1084 | } |
1085 | ||
c020395b | 1086 | static __always_inline bool |
aa525d06 | 1087 | atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1088 | { |
3570a1bc ME |
1089 | instrument_atomic_read_write(v, sizeof(*v)); |
1090 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 | 1091 | return arch_atomic64_try_cmpxchg_acquire(v, old, new); |
b06ed71a DV |
1092 | } |
1093 | ||
c020395b | 1094 | static __always_inline bool |
aa525d06 | 1095 | atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1096 | { |
3570a1bc ME |
1097 | instrument_atomic_read_write(v, sizeof(*v)); |
1098 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 | 1099 | return arch_atomic64_try_cmpxchg_release(v, old, new); |
b06ed71a DV |
1100 | } |
1101 | ||
c020395b | 1102 | static __always_inline bool |
aa525d06 | 1103 | atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1104 | { |
3570a1bc ME |
1105 | instrument_atomic_read_write(v, sizeof(*v)); |
1106 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 | 1107 | return arch_atomic64_try_cmpxchg_relaxed(v, old, new); |
b06ed71a DV |
1108 | } |
1109 | ||
c020395b | 1110 | static __always_inline bool |
aa525d06 | 1111 | atomic64_sub_and_test(s64 i, atomic64_t *v) |
b06ed71a | 1112 | { |
3570a1bc | 1113 | instrument_atomic_read_write(v, sizeof(*v)); |
b06ed71a DV |
1114 | return arch_atomic64_sub_and_test(i, v); |
1115 | } | |
1116 | ||
c020395b | 1117 | static __always_inline bool |
aa525d06 | 1118 | atomic64_dec_and_test(atomic64_t *v) |
b06ed71a | 1119 | { |
3570a1bc | 1120 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1121 | return arch_atomic64_dec_and_test(v); |
b06ed71a DV |
1122 | } |
1123 | ||
c020395b | 1124 | static __always_inline bool |
aa525d06 MR |
1125 | atomic64_inc_and_test(atomic64_t *v) |
1126 | { | |
3570a1bc | 1127 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1128 | return arch_atomic64_inc_and_test(v); |
1129 | } | |
aa525d06 | 1130 | |
c020395b | 1131 | static __always_inline bool |
aa525d06 | 1132 | atomic64_add_negative(s64 i, atomic64_t *v) |
b06ed71a | 1133 | { |
3570a1bc | 1134 | instrument_atomic_read_write(v, sizeof(*v)); |
b06ed71a DV |
1135 | return arch_atomic64_add_negative(i, v); |
1136 | } | |
aa525d06 | 1137 | |
c020395b | 1138 | static __always_inline s64 |
aa525d06 MR |
1139 | atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) |
1140 | { | |
3570a1bc | 1141 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1142 | return arch_atomic64_fetch_add_unless(v, a, u); |
1143 | } | |
aa525d06 | 1144 | |
c020395b | 1145 | static __always_inline bool |
aa525d06 MR |
1146 | atomic64_add_unless(atomic64_t *v, s64 a, s64 u) |
1147 | { | |
3570a1bc | 1148 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1149 | return arch_atomic64_add_unless(v, a, u); |
1150 | } | |
aa525d06 | 1151 | |
c020395b | 1152 | static __always_inline bool |
aa525d06 MR |
1153 | atomic64_inc_not_zero(atomic64_t *v) |
1154 | { | |
3570a1bc | 1155 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1156 | return arch_atomic64_inc_not_zero(v); |
1157 | } | |
aa525d06 | 1158 | |
c020395b | 1159 | static __always_inline bool |
aa525d06 MR |
1160 | atomic64_inc_unless_negative(atomic64_t *v) |
1161 | { | |
3570a1bc | 1162 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1163 | return arch_atomic64_inc_unless_negative(v); |
1164 | } | |
b06ed71a | 1165 | |
c020395b | 1166 | static __always_inline bool |
aa525d06 MR |
1167 | atomic64_dec_unless_positive(atomic64_t *v) |
1168 | { | |
3570a1bc | 1169 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1170 | return arch_atomic64_dec_unless_positive(v); |
1171 | } | |
aa525d06 | 1172 | |
c020395b | 1173 | static __always_inline s64 |
aa525d06 MR |
1174 | atomic64_dec_if_positive(atomic64_t *v) |
1175 | { | |
3570a1bc | 1176 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1177 | return arch_atomic64_dec_if_positive(v); |
1178 | } | |
aa525d06 | 1179 | |
29f006fd PZ |
1180 | #define xchg(ptr, ...) \ |
1181 | ({ \ | |
1182 | typeof(ptr) __ai_ptr = (ptr); \ | |
1183 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1184 | arch_xchg(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1185 | }) |
aa525d06 | 1186 | |
29f006fd PZ |
1187 | #define xchg_acquire(ptr, ...) \ |
1188 | ({ \ | |
1189 | typeof(ptr) __ai_ptr = (ptr); \ | |
1190 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1191 | arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1192 | }) |
aa525d06 | 1193 | |
29f006fd PZ |
1194 | #define xchg_release(ptr, ...) \ |
1195 | ({ \ | |
1196 | typeof(ptr) __ai_ptr = (ptr); \ | |
1197 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1198 | arch_xchg_release(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1199 | }) |
aa525d06 | 1200 | |
29f006fd PZ |
1201 | #define xchg_relaxed(ptr, ...) \ |
1202 | ({ \ | |
1203 | typeof(ptr) __ai_ptr = (ptr); \ | |
1204 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1205 | arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1206 | }) |
aa525d06 | 1207 | |
29f006fd PZ |
1208 | #define cmpxchg(ptr, ...) \ |
1209 | ({ \ | |
1210 | typeof(ptr) __ai_ptr = (ptr); \ | |
1211 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1212 | arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ | |
f9881cc4 | 1213 | }) |
aa525d06 | 1214 | |
29f006fd PZ |
1215 | #define cmpxchg_acquire(ptr, ...) \ |
1216 | ({ \ | |
1217 | typeof(ptr) __ai_ptr = (ptr); \ | |
1218 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1219 | arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1220 | }) |
aa525d06 | 1221 | |
29f006fd PZ |
1222 | #define cmpxchg_release(ptr, ...) \ |
1223 | ({ \ | |
1224 | typeof(ptr) __ai_ptr = (ptr); \ | |
1225 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1226 | arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1227 | }) |
aa525d06 | 1228 | |
29f006fd PZ |
1229 | #define cmpxchg_relaxed(ptr, ...) \ |
1230 | ({ \ | |
1231 | typeof(ptr) __ai_ptr = (ptr); \ | |
1232 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1233 | arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1234 | }) |
f9881cc4 | 1235 | |
29f006fd PZ |
1236 | #define cmpxchg64(ptr, ...) \ |
1237 | ({ \ | |
1238 | typeof(ptr) __ai_ptr = (ptr); \ | |
1239 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1240 | arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a | 1241 | }) |
aa525d06 | 1242 | |
29f006fd PZ |
1243 | #define cmpxchg64_acquire(ptr, ...) \ |
1244 | ({ \ | |
1245 | typeof(ptr) __ai_ptr = (ptr); \ | |
1246 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1247 | arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1248 | }) |
aa525d06 | 1249 | |
29f006fd PZ |
1250 | #define cmpxchg64_release(ptr, ...) \ |
1251 | ({ \ | |
1252 | typeof(ptr) __ai_ptr = (ptr); \ | |
1253 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1254 | arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1255 | }) |
b06ed71a | 1256 | |
29f006fd PZ |
1257 | #define cmpxchg64_relaxed(ptr, ...) \ |
1258 | ({ \ | |
1259 | typeof(ptr) __ai_ptr = (ptr); \ | |
1260 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1261 | arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a DV |
1262 | }) |
1263 | ||
29f006fd PZ |
1264 | #define try_cmpxchg(ptr, oldp, ...) \ |
1265 | ({ \ | |
1266 | typeof(ptr) __ai_ptr = (ptr); \ | |
1267 | typeof(oldp) __ai_oldp = (oldp); \ | |
1268 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1269 | instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
1270 | arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ | |
1271 | }) | |
29f006fd | 1272 | |
29f006fd PZ |
1273 | #define try_cmpxchg_acquire(ptr, oldp, ...) \ |
1274 | ({ \ | |
1275 | typeof(ptr) __ai_ptr = (ptr); \ | |
1276 | typeof(oldp) __ai_oldp = (oldp); \ | |
1277 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1278 | instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
1279 | arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ | |
1280 | }) | |
29f006fd | 1281 | |
29f006fd PZ |
1282 | #define try_cmpxchg_release(ptr, oldp, ...) \ |
1283 | ({ \ | |
1284 | typeof(ptr) __ai_ptr = (ptr); \ | |
1285 | typeof(oldp) __ai_oldp = (oldp); \ | |
1286 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1287 | instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
1288 | arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ | |
1289 | }) | |
29f006fd | 1290 | |
29f006fd PZ |
1291 | #define try_cmpxchg_relaxed(ptr, oldp, ...) \ |
1292 | ({ \ | |
1293 | typeof(ptr) __ai_ptr = (ptr); \ | |
1294 | typeof(oldp) __ai_oldp = (oldp); \ | |
1295 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1296 | instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
1297 | arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ | |
1298 | }) | |
29f006fd PZ |
1299 | |
1300 | #define cmpxchg_local(ptr, ...) \ | |
1301 | ({ \ | |
1302 | typeof(ptr) __ai_ptr = (ptr); \ | |
1303 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1304 | arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a DV |
1305 | }) |
1306 | ||
29f006fd PZ |
1307 | #define cmpxchg64_local(ptr, ...) \ |
1308 | ({ \ | |
1309 | typeof(ptr) __ai_ptr = (ptr); \ | |
1310 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1311 | arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a DV |
1312 | }) |
1313 | ||
29f006fd PZ |
1314 | #define sync_cmpxchg(ptr, ...) \ |
1315 | ({ \ | |
1316 | typeof(ptr) __ai_ptr = (ptr); \ | |
1317 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1318 | arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a DV |
1319 | }) |
1320 | ||
29f006fd PZ |
1321 | #define cmpxchg_double(ptr, ...) \ |
1322 | ({ \ | |
1323 | typeof(ptr) __ai_ptr = (ptr); \ | |
1324 | instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ | |
1325 | arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a DV |
1326 | }) |
1327 | ||
aa525d06 | 1328 | |
29f006fd PZ |
1329 | #define cmpxchg_double_local(ptr, ...) \ |
1330 | ({ \ | |
1331 | typeof(ptr) __ai_ptr = (ptr); \ | |
1332 | instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ | |
1333 | arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a DV |
1334 | }) |
1335 | ||
aa525d06 | 1336 | #endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */ |
bccf1ec3 | 1337 | // 1d7c3a25aca5c7fb031c307be4c3d24c7b48fcd5 |