Commit | Line | Data |
---|---|---|
aa525d06 MR |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | // Generated by scripts/atomic/gen-atomic-instrumented.sh | |
4 | // DO NOT MODIFY THIS FILE DIRECTLY | |
5 | ||
ac605bee DV |
6 | /* |
7 | * This file provides wrappers with KASAN instrumentation for atomic operations. | |
8 | * To use this functionality an arch's atomic.h file needs to define all | |
9 | * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include | |
10 | * this file at the end. This file provides atomic_read() that forwards to | |
11 | * arch_atomic_read() for actual atomic operation. | |
12 | * Note: if an arch atomic operation is implemented by means of other atomic | |
13 | * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use | |
14 | * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid | |
15 | * double instrumentation. | |
16 | */ | |
e3d18cee MR |
17 | #ifndef _LINUX_ATOMIC_INSTRUMENTED_H |
18 | #define _LINUX_ATOMIC_INSTRUMENTED_H | |
aa525d06 MR |
19 | |
20 | #include <linux/build_bug.h> | |
c020395b | 21 | #include <linux/compiler.h> |
ed8af2e4 | 22 | #include <linux/instrumented.h> |
aa525d06 | 23 | |
c020395b | 24 | static __always_inline int |
aa525d06 MR |
25 | atomic_read(const atomic_t *v) |
26 | { | |
ed8af2e4 | 27 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
28 | return arch_atomic_read(v); |
29 | } | |
aa525d06 | 30 | |
c020395b | 31 | static __always_inline int |
aa525d06 MR |
32 | atomic_read_acquire(const atomic_t *v) |
33 | { | |
ed8af2e4 | 34 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
35 | return arch_atomic_read_acquire(v); |
36 | } | |
aa525d06 | 37 | |
c020395b | 38 | static __always_inline void |
aa525d06 MR |
39 | atomic_set(atomic_t *v, int i) |
40 | { | |
ed8af2e4 | 41 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
42 | arch_atomic_set(v, i); |
43 | } | |
aa525d06 | 44 | |
c020395b | 45 | static __always_inline void |
aa525d06 MR |
46 | atomic_set_release(atomic_t *v, int i) |
47 | { | |
e87c4f66 | 48 | kcsan_release(); |
ed8af2e4 | 49 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
50 | arch_atomic_set_release(v, i); |
51 | } | |
aa525d06 | 52 | |
c020395b | 53 | static __always_inline void |
aa525d06 MR |
54 | atomic_add(int i, atomic_t *v) |
55 | { | |
3570a1bc | 56 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
57 | arch_atomic_add(i, v); |
58 | } | |
aa525d06 | 59 | |
c020395b | 60 | static __always_inline int |
aa525d06 MR |
61 | atomic_add_return(int i, atomic_t *v) |
62 | { | |
e87c4f66 | 63 | kcsan_mb(); |
3570a1bc | 64 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
65 | return arch_atomic_add_return(i, v); |
66 | } | |
aa525d06 | 67 | |
c020395b | 68 | static __always_inline int |
aa525d06 MR |
69 | atomic_add_return_acquire(int i, atomic_t *v) |
70 | { | |
3570a1bc | 71 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
72 | return arch_atomic_add_return_acquire(i, v); |
73 | } | |
aa525d06 | 74 | |
c020395b | 75 | static __always_inline int |
aa525d06 MR |
76 | atomic_add_return_release(int i, atomic_t *v) |
77 | { | |
e87c4f66 | 78 | kcsan_release(); |
3570a1bc | 79 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
80 | return arch_atomic_add_return_release(i, v); |
81 | } | |
aa525d06 | 82 | |
c020395b | 83 | static __always_inline int |
aa525d06 MR |
84 | atomic_add_return_relaxed(int i, atomic_t *v) |
85 | { | |
3570a1bc | 86 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
87 | return arch_atomic_add_return_relaxed(i, v); |
88 | } | |
aa525d06 | 89 | |
c020395b | 90 | static __always_inline int |
aa525d06 MR |
91 | atomic_fetch_add(int i, atomic_t *v) |
92 | { | |
e87c4f66 | 93 | kcsan_mb(); |
3570a1bc | 94 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
95 | return arch_atomic_fetch_add(i, v); |
96 | } | |
aa525d06 | 97 | |
c020395b | 98 | static __always_inline int |
aa525d06 MR |
99 | atomic_fetch_add_acquire(int i, atomic_t *v) |
100 | { | |
3570a1bc | 101 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
102 | return arch_atomic_fetch_add_acquire(i, v); |
103 | } | |
aa525d06 | 104 | |
c020395b | 105 | static __always_inline int |
aa525d06 MR |
106 | atomic_fetch_add_release(int i, atomic_t *v) |
107 | { | |
e87c4f66 | 108 | kcsan_release(); |
3570a1bc | 109 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
110 | return arch_atomic_fetch_add_release(i, v); |
111 | } | |
aa525d06 | 112 | |
c020395b | 113 | static __always_inline int |
aa525d06 MR |
114 | atomic_fetch_add_relaxed(int i, atomic_t *v) |
115 | { | |
3570a1bc | 116 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
117 | return arch_atomic_fetch_add_relaxed(i, v); |
118 | } | |
aa525d06 | 119 | |
c020395b | 120 | static __always_inline void |
aa525d06 MR |
121 | atomic_sub(int i, atomic_t *v) |
122 | { | |
3570a1bc | 123 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
124 | arch_atomic_sub(i, v); |
125 | } | |
aa525d06 | 126 | |
c020395b | 127 | static __always_inline int |
aa525d06 MR |
128 | atomic_sub_return(int i, atomic_t *v) |
129 | { | |
e87c4f66 | 130 | kcsan_mb(); |
3570a1bc | 131 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
132 | return arch_atomic_sub_return(i, v); |
133 | } | |
aa525d06 | 134 | |
c020395b | 135 | static __always_inline int |
aa525d06 MR |
136 | atomic_sub_return_acquire(int i, atomic_t *v) |
137 | { | |
3570a1bc | 138 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
139 | return arch_atomic_sub_return_acquire(i, v); |
140 | } | |
aa525d06 | 141 | |
c020395b | 142 | static __always_inline int |
aa525d06 MR |
143 | atomic_sub_return_release(int i, atomic_t *v) |
144 | { | |
e87c4f66 | 145 | kcsan_release(); |
3570a1bc | 146 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
147 | return arch_atomic_sub_return_release(i, v); |
148 | } | |
aa525d06 | 149 | |
c020395b | 150 | static __always_inline int |
aa525d06 MR |
151 | atomic_sub_return_relaxed(int i, atomic_t *v) |
152 | { | |
3570a1bc | 153 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
154 | return arch_atomic_sub_return_relaxed(i, v); |
155 | } | |
aa525d06 | 156 | |
c020395b | 157 | static __always_inline int |
aa525d06 MR |
158 | atomic_fetch_sub(int i, atomic_t *v) |
159 | { | |
e87c4f66 | 160 | kcsan_mb(); |
3570a1bc | 161 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
162 | return arch_atomic_fetch_sub(i, v); |
163 | } | |
aa525d06 | 164 | |
c020395b | 165 | static __always_inline int |
aa525d06 MR |
166 | atomic_fetch_sub_acquire(int i, atomic_t *v) |
167 | { | |
3570a1bc | 168 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
169 | return arch_atomic_fetch_sub_acquire(i, v); |
170 | } | |
aa525d06 | 171 | |
c020395b | 172 | static __always_inline int |
aa525d06 MR |
173 | atomic_fetch_sub_release(int i, atomic_t *v) |
174 | { | |
e87c4f66 | 175 | kcsan_release(); |
3570a1bc | 176 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
177 | return arch_atomic_fetch_sub_release(i, v); |
178 | } | |
aa525d06 | 179 | |
c020395b | 180 | static __always_inline int |
aa525d06 MR |
181 | atomic_fetch_sub_relaxed(int i, atomic_t *v) |
182 | { | |
3570a1bc | 183 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
184 | return arch_atomic_fetch_sub_relaxed(i, v); |
185 | } | |
aa525d06 | 186 | |
c020395b | 187 | static __always_inline void |
aa525d06 MR |
188 | atomic_inc(atomic_t *v) |
189 | { | |
3570a1bc | 190 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
191 | arch_atomic_inc(v); |
192 | } | |
aa525d06 | 193 | |
c020395b | 194 | static __always_inline int |
aa525d06 MR |
195 | atomic_inc_return(atomic_t *v) |
196 | { | |
e87c4f66 | 197 | kcsan_mb(); |
3570a1bc | 198 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
199 | return arch_atomic_inc_return(v); |
200 | } | |
aa525d06 | 201 | |
c020395b | 202 | static __always_inline int |
aa525d06 MR |
203 | atomic_inc_return_acquire(atomic_t *v) |
204 | { | |
3570a1bc | 205 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
206 | return arch_atomic_inc_return_acquire(v); |
207 | } | |
aa525d06 | 208 | |
c020395b | 209 | static __always_inline int |
aa525d06 MR |
210 | atomic_inc_return_release(atomic_t *v) |
211 | { | |
e87c4f66 | 212 | kcsan_release(); |
3570a1bc | 213 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
214 | return arch_atomic_inc_return_release(v); |
215 | } | |
aa525d06 | 216 | |
c020395b | 217 | static __always_inline int |
aa525d06 MR |
218 | atomic_inc_return_relaxed(atomic_t *v) |
219 | { | |
3570a1bc | 220 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
221 | return arch_atomic_inc_return_relaxed(v); |
222 | } | |
aa525d06 | 223 | |
c020395b | 224 | static __always_inline int |
aa525d06 MR |
225 | atomic_fetch_inc(atomic_t *v) |
226 | { | |
e87c4f66 | 227 | kcsan_mb(); |
3570a1bc | 228 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
229 | return arch_atomic_fetch_inc(v); |
230 | } | |
aa525d06 | 231 | |
c020395b | 232 | static __always_inline int |
aa525d06 MR |
233 | atomic_fetch_inc_acquire(atomic_t *v) |
234 | { | |
3570a1bc | 235 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
236 | return arch_atomic_fetch_inc_acquire(v); |
237 | } | |
aa525d06 | 238 | |
c020395b | 239 | static __always_inline int |
aa525d06 MR |
240 | atomic_fetch_inc_release(atomic_t *v) |
241 | { | |
e87c4f66 | 242 | kcsan_release(); |
3570a1bc | 243 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
244 | return arch_atomic_fetch_inc_release(v); |
245 | } | |
aa525d06 | 246 | |
c020395b | 247 | static __always_inline int |
aa525d06 MR |
248 | atomic_fetch_inc_relaxed(atomic_t *v) |
249 | { | |
3570a1bc | 250 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
251 | return arch_atomic_fetch_inc_relaxed(v); |
252 | } | |
aa525d06 | 253 | |
c020395b | 254 | static __always_inline void |
aa525d06 MR |
255 | atomic_dec(atomic_t *v) |
256 | { | |
3570a1bc | 257 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
258 | arch_atomic_dec(v); |
259 | } | |
aa525d06 | 260 | |
c020395b | 261 | static __always_inline int |
aa525d06 MR |
262 | atomic_dec_return(atomic_t *v) |
263 | { | |
e87c4f66 | 264 | kcsan_mb(); |
3570a1bc | 265 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
266 | return arch_atomic_dec_return(v); |
267 | } | |
aa525d06 | 268 | |
c020395b | 269 | static __always_inline int |
aa525d06 MR |
270 | atomic_dec_return_acquire(atomic_t *v) |
271 | { | |
3570a1bc | 272 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
273 | return arch_atomic_dec_return_acquire(v); |
274 | } | |
aa525d06 | 275 | |
c020395b | 276 | static __always_inline int |
aa525d06 MR |
277 | atomic_dec_return_release(atomic_t *v) |
278 | { | |
e87c4f66 | 279 | kcsan_release(); |
3570a1bc | 280 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
281 | return arch_atomic_dec_return_release(v); |
282 | } | |
aa525d06 | 283 | |
c020395b | 284 | static __always_inline int |
aa525d06 MR |
285 | atomic_dec_return_relaxed(atomic_t *v) |
286 | { | |
3570a1bc | 287 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
288 | return arch_atomic_dec_return_relaxed(v); |
289 | } | |
aa525d06 | 290 | |
c020395b | 291 | static __always_inline int |
aa525d06 MR |
292 | atomic_fetch_dec(atomic_t *v) |
293 | { | |
e87c4f66 | 294 | kcsan_mb(); |
3570a1bc | 295 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
296 | return arch_atomic_fetch_dec(v); |
297 | } | |
aa525d06 | 298 | |
c020395b | 299 | static __always_inline int |
aa525d06 MR |
300 | atomic_fetch_dec_acquire(atomic_t *v) |
301 | { | |
3570a1bc | 302 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
303 | return arch_atomic_fetch_dec_acquire(v); |
304 | } | |
aa525d06 | 305 | |
c020395b | 306 | static __always_inline int |
aa525d06 MR |
307 | atomic_fetch_dec_release(atomic_t *v) |
308 | { | |
e87c4f66 | 309 | kcsan_release(); |
3570a1bc | 310 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
311 | return arch_atomic_fetch_dec_release(v); |
312 | } | |
aa525d06 | 313 | |
c020395b | 314 | static __always_inline int |
aa525d06 MR |
315 | atomic_fetch_dec_relaxed(atomic_t *v) |
316 | { | |
3570a1bc | 317 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
318 | return arch_atomic_fetch_dec_relaxed(v); |
319 | } | |
aa525d06 | 320 | |
c020395b | 321 | static __always_inline void |
aa525d06 MR |
322 | atomic_and(int i, atomic_t *v) |
323 | { | |
3570a1bc | 324 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
325 | arch_atomic_and(i, v); |
326 | } | |
aa525d06 | 327 | |
c020395b | 328 | static __always_inline int |
aa525d06 MR |
329 | atomic_fetch_and(int i, atomic_t *v) |
330 | { | |
e87c4f66 | 331 | kcsan_mb(); |
3570a1bc | 332 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
333 | return arch_atomic_fetch_and(i, v); |
334 | } | |
aa525d06 | 335 | |
c020395b | 336 | static __always_inline int |
aa525d06 MR |
337 | atomic_fetch_and_acquire(int i, atomic_t *v) |
338 | { | |
3570a1bc | 339 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
340 | return arch_atomic_fetch_and_acquire(i, v); |
341 | } | |
aa525d06 | 342 | |
c020395b | 343 | static __always_inline int |
aa525d06 MR |
344 | atomic_fetch_and_release(int i, atomic_t *v) |
345 | { | |
e87c4f66 | 346 | kcsan_release(); |
3570a1bc | 347 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
348 | return arch_atomic_fetch_and_release(i, v); |
349 | } | |
aa525d06 | 350 | |
c020395b | 351 | static __always_inline int |
aa525d06 MR |
352 | atomic_fetch_and_relaxed(int i, atomic_t *v) |
353 | { | |
3570a1bc | 354 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
355 | return arch_atomic_fetch_and_relaxed(i, v); |
356 | } | |
aa525d06 | 357 | |
c020395b | 358 | static __always_inline void |
aa525d06 MR |
359 | atomic_andnot(int i, atomic_t *v) |
360 | { | |
3570a1bc | 361 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
362 | arch_atomic_andnot(i, v); |
363 | } | |
aa525d06 | 364 | |
c020395b | 365 | static __always_inline int |
aa525d06 MR |
366 | atomic_fetch_andnot(int i, atomic_t *v) |
367 | { | |
e87c4f66 | 368 | kcsan_mb(); |
3570a1bc | 369 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
370 | return arch_atomic_fetch_andnot(i, v); |
371 | } | |
aa525d06 | 372 | |
c020395b | 373 | static __always_inline int |
aa525d06 MR |
374 | atomic_fetch_andnot_acquire(int i, atomic_t *v) |
375 | { | |
3570a1bc | 376 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
377 | return arch_atomic_fetch_andnot_acquire(i, v); |
378 | } | |
aa525d06 | 379 | |
c020395b | 380 | static __always_inline int |
aa525d06 MR |
381 | atomic_fetch_andnot_release(int i, atomic_t *v) |
382 | { | |
e87c4f66 | 383 | kcsan_release(); |
3570a1bc | 384 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
385 | return arch_atomic_fetch_andnot_release(i, v); |
386 | } | |
aa525d06 | 387 | |
c020395b | 388 | static __always_inline int |
aa525d06 MR |
389 | atomic_fetch_andnot_relaxed(int i, atomic_t *v) |
390 | { | |
3570a1bc | 391 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
392 | return arch_atomic_fetch_andnot_relaxed(i, v); |
393 | } | |
aa525d06 | 394 | |
c020395b | 395 | static __always_inline void |
aa525d06 MR |
396 | atomic_or(int i, atomic_t *v) |
397 | { | |
3570a1bc | 398 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
399 | arch_atomic_or(i, v); |
400 | } | |
aa525d06 | 401 | |
c020395b | 402 | static __always_inline int |
aa525d06 MR |
403 | atomic_fetch_or(int i, atomic_t *v) |
404 | { | |
e87c4f66 | 405 | kcsan_mb(); |
3570a1bc | 406 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
407 | return arch_atomic_fetch_or(i, v); |
408 | } | |
aa525d06 | 409 | |
c020395b | 410 | static __always_inline int |
aa525d06 MR |
411 | atomic_fetch_or_acquire(int i, atomic_t *v) |
412 | { | |
3570a1bc | 413 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
414 | return arch_atomic_fetch_or_acquire(i, v); |
415 | } | |
aa525d06 | 416 | |
c020395b | 417 | static __always_inline int |
aa525d06 MR |
418 | atomic_fetch_or_release(int i, atomic_t *v) |
419 | { | |
e87c4f66 | 420 | kcsan_release(); |
3570a1bc | 421 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
422 | return arch_atomic_fetch_or_release(i, v); |
423 | } | |
aa525d06 | 424 | |
c020395b | 425 | static __always_inline int |
aa525d06 MR |
426 | atomic_fetch_or_relaxed(int i, atomic_t *v) |
427 | { | |
3570a1bc | 428 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
429 | return arch_atomic_fetch_or_relaxed(i, v); |
430 | } | |
aa525d06 | 431 | |
c020395b | 432 | static __always_inline void |
aa525d06 MR |
433 | atomic_xor(int i, atomic_t *v) |
434 | { | |
3570a1bc | 435 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
436 | arch_atomic_xor(i, v); |
437 | } | |
aa525d06 | 438 | |
c020395b | 439 | static __always_inline int |
aa525d06 MR |
440 | atomic_fetch_xor(int i, atomic_t *v) |
441 | { | |
e87c4f66 | 442 | kcsan_mb(); |
3570a1bc | 443 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
444 | return arch_atomic_fetch_xor(i, v); |
445 | } | |
aa525d06 | 446 | |
c020395b | 447 | static __always_inline int |
aa525d06 MR |
448 | atomic_fetch_xor_acquire(int i, atomic_t *v) |
449 | { | |
3570a1bc | 450 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
451 | return arch_atomic_fetch_xor_acquire(i, v); |
452 | } | |
aa525d06 | 453 | |
c020395b | 454 | static __always_inline int |
aa525d06 MR |
455 | atomic_fetch_xor_release(int i, atomic_t *v) |
456 | { | |
e87c4f66 | 457 | kcsan_release(); |
3570a1bc | 458 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
459 | return arch_atomic_fetch_xor_release(i, v); |
460 | } | |
aa525d06 | 461 | |
c020395b | 462 | static __always_inline int |
aa525d06 MR |
463 | atomic_fetch_xor_relaxed(int i, atomic_t *v) |
464 | { | |
3570a1bc | 465 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
466 | return arch_atomic_fetch_xor_relaxed(i, v); |
467 | } | |
aa525d06 | 468 | |
c020395b | 469 | static __always_inline int |
aa525d06 MR |
470 | atomic_xchg(atomic_t *v, int i) |
471 | { | |
e87c4f66 | 472 | kcsan_mb(); |
3570a1bc | 473 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
474 | return arch_atomic_xchg(v, i); |
475 | } | |
aa525d06 | 476 | |
c020395b | 477 | static __always_inline int |
aa525d06 MR |
478 | atomic_xchg_acquire(atomic_t *v, int i) |
479 | { | |
3570a1bc | 480 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
481 | return arch_atomic_xchg_acquire(v, i); |
482 | } | |
aa525d06 | 483 | |
c020395b | 484 | static __always_inline int |
aa525d06 MR |
485 | atomic_xchg_release(atomic_t *v, int i) |
486 | { | |
e87c4f66 | 487 | kcsan_release(); |
3570a1bc | 488 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
489 | return arch_atomic_xchg_release(v, i); |
490 | } | |
aa525d06 | 491 | |
c020395b | 492 | static __always_inline int |
aa525d06 MR |
493 | atomic_xchg_relaxed(atomic_t *v, int i) |
494 | { | |
3570a1bc | 495 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
496 | return arch_atomic_xchg_relaxed(v, i); |
497 | } | |
aa525d06 | 498 | |
c020395b | 499 | static __always_inline int |
aa525d06 MR |
500 | atomic_cmpxchg(atomic_t *v, int old, int new) |
501 | { | |
e87c4f66 | 502 | kcsan_mb(); |
3570a1bc | 503 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
504 | return arch_atomic_cmpxchg(v, old, new); |
505 | } | |
aa525d06 | 506 | |
c020395b | 507 | static __always_inline int |
aa525d06 MR |
508 | atomic_cmpxchg_acquire(atomic_t *v, int old, int new) |
509 | { | |
3570a1bc | 510 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
511 | return arch_atomic_cmpxchg_acquire(v, old, new); |
512 | } | |
aa525d06 | 513 | |
c020395b | 514 | static __always_inline int |
aa525d06 MR |
515 | atomic_cmpxchg_release(atomic_t *v, int old, int new) |
516 | { | |
e87c4f66 | 517 | kcsan_release(); |
3570a1bc | 518 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
519 | return arch_atomic_cmpxchg_release(v, old, new); |
520 | } | |
aa525d06 | 521 | |
c020395b | 522 | static __always_inline int |
aa525d06 MR |
523 | atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) |
524 | { | |
3570a1bc | 525 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
526 | return arch_atomic_cmpxchg_relaxed(v, old, new); |
527 | } | |
aa525d06 | 528 | |
c020395b | 529 | static __always_inline bool |
aa525d06 MR |
530 | atomic_try_cmpxchg(atomic_t *v, int *old, int new) |
531 | { | |
e87c4f66 | 532 | kcsan_mb(); |
3570a1bc ME |
533 | instrument_atomic_read_write(v, sizeof(*v)); |
534 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 MR |
535 | return arch_atomic_try_cmpxchg(v, old, new); |
536 | } | |
aa525d06 | 537 | |
c020395b | 538 | static __always_inline bool |
aa525d06 MR |
539 | atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) |
540 | { | |
3570a1bc ME |
541 | instrument_atomic_read_write(v, sizeof(*v)); |
542 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 MR |
543 | return arch_atomic_try_cmpxchg_acquire(v, old, new); |
544 | } | |
aa525d06 | 545 | |
c020395b | 546 | static __always_inline bool |
aa525d06 MR |
547 | atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) |
548 | { | |
e87c4f66 | 549 | kcsan_release(); |
3570a1bc ME |
550 | instrument_atomic_read_write(v, sizeof(*v)); |
551 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 MR |
552 | return arch_atomic_try_cmpxchg_release(v, old, new); |
553 | } | |
aa525d06 | 554 | |
c020395b | 555 | static __always_inline bool |
aa525d06 MR |
556 | atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) |
557 | { | |
3570a1bc ME |
558 | instrument_atomic_read_write(v, sizeof(*v)); |
559 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 MR |
560 | return arch_atomic_try_cmpxchg_relaxed(v, old, new); |
561 | } | |
aa525d06 | 562 | |
c020395b | 563 | static __always_inline bool |
aa525d06 MR |
564 | atomic_sub_and_test(int i, atomic_t *v) |
565 | { | |
e87c4f66 | 566 | kcsan_mb(); |
3570a1bc | 567 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
568 | return arch_atomic_sub_and_test(i, v); |
569 | } | |
aa525d06 | 570 | |
c020395b | 571 | static __always_inline bool |
aa525d06 MR |
572 | atomic_dec_and_test(atomic_t *v) |
573 | { | |
e87c4f66 | 574 | kcsan_mb(); |
3570a1bc | 575 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
576 | return arch_atomic_dec_and_test(v); |
577 | } | |
aa525d06 | 578 | |
c020395b | 579 | static __always_inline bool |
aa525d06 MR |
580 | atomic_inc_and_test(atomic_t *v) |
581 | { | |
e87c4f66 | 582 | kcsan_mb(); |
3570a1bc | 583 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
584 | return arch_atomic_inc_and_test(v); |
585 | } | |
aa525d06 | 586 | |
c020395b | 587 | static __always_inline bool |
aa525d06 MR |
588 | atomic_add_negative(int i, atomic_t *v) |
589 | { | |
e87c4f66 | 590 | kcsan_mb(); |
3570a1bc | 591 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
592 | return arch_atomic_add_negative(i, v); |
593 | } | |
aa525d06 | 594 | |
c020395b | 595 | static __always_inline int |
aa525d06 MR |
596 | atomic_fetch_add_unless(atomic_t *v, int a, int u) |
597 | { | |
e87c4f66 | 598 | kcsan_mb(); |
3570a1bc | 599 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
600 | return arch_atomic_fetch_add_unless(v, a, u); |
601 | } | |
aa525d06 | 602 | |
c020395b | 603 | static __always_inline bool |
aa525d06 MR |
604 | atomic_add_unless(atomic_t *v, int a, int u) |
605 | { | |
e87c4f66 | 606 | kcsan_mb(); |
3570a1bc | 607 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
608 | return arch_atomic_add_unless(v, a, u); |
609 | } | |
aa525d06 | 610 | |
c020395b | 611 | static __always_inline bool |
aa525d06 MR |
612 | atomic_inc_not_zero(atomic_t *v) |
613 | { | |
e87c4f66 | 614 | kcsan_mb(); |
3570a1bc | 615 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
616 | return arch_atomic_inc_not_zero(v); |
617 | } | |
aa525d06 | 618 | |
c020395b | 619 | static __always_inline bool |
aa525d06 MR |
620 | atomic_inc_unless_negative(atomic_t *v) |
621 | { | |
e87c4f66 | 622 | kcsan_mb(); |
3570a1bc | 623 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
624 | return arch_atomic_inc_unless_negative(v); |
625 | } | |
aa525d06 | 626 | |
c020395b | 627 | static __always_inline bool |
aa525d06 MR |
628 | atomic_dec_unless_positive(atomic_t *v) |
629 | { | |
e87c4f66 | 630 | kcsan_mb(); |
3570a1bc | 631 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
632 | return arch_atomic_dec_unless_positive(v); |
633 | } | |
aa525d06 | 634 | |
c020395b | 635 | static __always_inline int |
aa525d06 MR |
636 | atomic_dec_if_positive(atomic_t *v) |
637 | { | |
e87c4f66 | 638 | kcsan_mb(); |
3570a1bc | 639 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
640 | return arch_atomic_dec_if_positive(v); |
641 | } | |
aa525d06 | 642 | |
c020395b | 643 | static __always_inline s64 |
aa525d06 MR |
644 | atomic64_read(const atomic64_t *v) |
645 | { | |
ed8af2e4 | 646 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
647 | return arch_atomic64_read(v); |
648 | } | |
aa525d06 | 649 | |
c020395b | 650 | static __always_inline s64 |
aa525d06 MR |
651 | atomic64_read_acquire(const atomic64_t *v) |
652 | { | |
ed8af2e4 | 653 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
654 | return arch_atomic64_read_acquire(v); |
655 | } | |
aa525d06 | 656 | |
c020395b | 657 | static __always_inline void |
aa525d06 MR |
658 | atomic64_set(atomic64_t *v, s64 i) |
659 | { | |
ed8af2e4 | 660 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
661 | arch_atomic64_set(v, i); |
662 | } | |
aa525d06 | 663 | |
c020395b | 664 | static __always_inline void |
aa525d06 MR |
665 | atomic64_set_release(atomic64_t *v, s64 i) |
666 | { | |
e87c4f66 | 667 | kcsan_release(); |
ed8af2e4 | 668 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
669 | arch_atomic64_set_release(v, i); |
670 | } | |
aa525d06 | 671 | |
c020395b | 672 | static __always_inline void |
aa525d06 MR |
673 | atomic64_add(s64 i, atomic64_t *v) |
674 | { | |
3570a1bc | 675 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
676 | arch_atomic64_add(i, v); |
677 | } | |
aa525d06 | 678 | |
c020395b | 679 | static __always_inline s64 |
aa525d06 MR |
680 | atomic64_add_return(s64 i, atomic64_t *v) |
681 | { | |
e87c4f66 | 682 | kcsan_mb(); |
3570a1bc | 683 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
684 | return arch_atomic64_add_return(i, v); |
685 | } | |
aa525d06 | 686 | |
c020395b | 687 | static __always_inline s64 |
aa525d06 MR |
688 | atomic64_add_return_acquire(s64 i, atomic64_t *v) |
689 | { | |
3570a1bc | 690 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
691 | return arch_atomic64_add_return_acquire(i, v); |
692 | } | |
aa525d06 | 693 | |
c020395b | 694 | static __always_inline s64 |
aa525d06 MR |
695 | atomic64_add_return_release(s64 i, atomic64_t *v) |
696 | { | |
e87c4f66 | 697 | kcsan_release(); |
3570a1bc | 698 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
699 | return arch_atomic64_add_return_release(i, v); |
700 | } | |
aa525d06 | 701 | |
c020395b | 702 | static __always_inline s64 |
aa525d06 MR |
703 | atomic64_add_return_relaxed(s64 i, atomic64_t *v) |
704 | { | |
3570a1bc | 705 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
706 | return arch_atomic64_add_return_relaxed(i, v); |
707 | } | |
aa525d06 | 708 | |
c020395b | 709 | static __always_inline s64 |
aa525d06 MR |
710 | atomic64_fetch_add(s64 i, atomic64_t *v) |
711 | { | |
e87c4f66 | 712 | kcsan_mb(); |
3570a1bc | 713 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
714 | return arch_atomic64_fetch_add(i, v); |
715 | } | |
aa525d06 | 716 | |
c020395b | 717 | static __always_inline s64 |
aa525d06 MR |
718 | atomic64_fetch_add_acquire(s64 i, atomic64_t *v) |
719 | { | |
3570a1bc | 720 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
721 | return arch_atomic64_fetch_add_acquire(i, v); |
722 | } | |
aa525d06 | 723 | |
c020395b | 724 | static __always_inline s64 |
aa525d06 MR |
725 | atomic64_fetch_add_release(s64 i, atomic64_t *v) |
726 | { | |
e87c4f66 | 727 | kcsan_release(); |
3570a1bc | 728 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
729 | return arch_atomic64_fetch_add_release(i, v); |
730 | } | |
aa525d06 | 731 | |
c020395b | 732 | static __always_inline s64 |
aa525d06 MR |
733 | atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) |
734 | { | |
3570a1bc | 735 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
736 | return arch_atomic64_fetch_add_relaxed(i, v); |
737 | } | |
aa525d06 | 738 | |
c020395b | 739 | static __always_inline void |
aa525d06 MR |
740 | atomic64_sub(s64 i, atomic64_t *v) |
741 | { | |
3570a1bc | 742 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
743 | arch_atomic64_sub(i, v); |
744 | } | |
ac605bee | 745 | |
c020395b | 746 | static __always_inline s64 |
aa525d06 MR |
747 | atomic64_sub_return(s64 i, atomic64_t *v) |
748 | { | |
e87c4f66 | 749 | kcsan_mb(); |
3570a1bc | 750 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
751 | return arch_atomic64_sub_return(i, v); |
752 | } | |
b06ed71a | 753 | |
c020395b | 754 | static __always_inline s64 |
aa525d06 MR |
755 | atomic64_sub_return_acquire(s64 i, atomic64_t *v) |
756 | { | |
3570a1bc | 757 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
758 | return arch_atomic64_sub_return_acquire(i, v); |
759 | } | |
b06ed71a | 760 | |
c020395b | 761 | static __always_inline s64 |
aa525d06 | 762 | atomic64_sub_return_release(s64 i, atomic64_t *v) |
b06ed71a | 763 | { |
e87c4f66 | 764 | kcsan_release(); |
3570a1bc | 765 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 766 | return arch_atomic64_sub_return_release(i, v); |
b06ed71a DV |
767 | } |
768 | ||
c020395b | 769 | static __always_inline s64 |
aa525d06 | 770 | atomic64_sub_return_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 771 | { |
3570a1bc | 772 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 773 | return arch_atomic64_sub_return_relaxed(i, v); |
b06ed71a DV |
774 | } |
775 | ||
c020395b | 776 | static __always_inline s64 |
aa525d06 | 777 | atomic64_fetch_sub(s64 i, atomic64_t *v) |
b06ed71a | 778 | { |
e87c4f66 | 779 | kcsan_mb(); |
3570a1bc | 780 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 781 | return arch_atomic64_fetch_sub(i, v); |
b06ed71a DV |
782 | } |
783 | ||
c020395b | 784 | static __always_inline s64 |
aa525d06 | 785 | atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) |
b06ed71a | 786 | { |
3570a1bc | 787 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 788 | return arch_atomic64_fetch_sub_acquire(i, v); |
b06ed71a DV |
789 | } |
790 | ||
c020395b | 791 | static __always_inline s64 |
aa525d06 | 792 | atomic64_fetch_sub_release(s64 i, atomic64_t *v) |
b06ed71a | 793 | { |
e87c4f66 | 794 | kcsan_release(); |
3570a1bc | 795 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 796 | return arch_atomic64_fetch_sub_release(i, v); |
b06ed71a DV |
797 | } |
798 | ||
c020395b | 799 | static __always_inline s64 |
aa525d06 | 800 | atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 801 | { |
3570a1bc | 802 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 803 | return arch_atomic64_fetch_sub_relaxed(i, v); |
b06ed71a DV |
804 | } |
805 | ||
c020395b | 806 | static __always_inline void |
aa525d06 | 807 | atomic64_inc(atomic64_t *v) |
b06ed71a | 808 | { |
3570a1bc | 809 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 810 | arch_atomic64_inc(v); |
b06ed71a DV |
811 | } |
812 | ||
c020395b | 813 | static __always_inline s64 |
aa525d06 | 814 | atomic64_inc_return(atomic64_t *v) |
b06ed71a | 815 | { |
e87c4f66 | 816 | kcsan_mb(); |
3570a1bc | 817 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 818 | return arch_atomic64_inc_return(v); |
b06ed71a DV |
819 | } |
820 | ||
c020395b | 821 | static __always_inline s64 |
aa525d06 | 822 | atomic64_inc_return_acquire(atomic64_t *v) |
b06ed71a | 823 | { |
3570a1bc | 824 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 825 | return arch_atomic64_inc_return_acquire(v); |
b06ed71a | 826 | } |
b06ed71a | 827 | |
c020395b | 828 | static __always_inline s64 |
aa525d06 | 829 | atomic64_inc_return_release(atomic64_t *v) |
b06ed71a | 830 | { |
e87c4f66 | 831 | kcsan_release(); |
3570a1bc | 832 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 833 | return arch_atomic64_inc_return_release(v); |
b06ed71a | 834 | } |
b06ed71a | 835 | |
c020395b | 836 | static __always_inline s64 |
aa525d06 | 837 | atomic64_inc_return_relaxed(atomic64_t *v) |
b06ed71a | 838 | { |
3570a1bc | 839 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 840 | return arch_atomic64_inc_return_relaxed(v); |
b06ed71a | 841 | } |
b06ed71a | 842 | |
c020395b | 843 | static __always_inline s64 |
aa525d06 | 844 | atomic64_fetch_inc(atomic64_t *v) |
0ae1d994 | 845 | { |
e87c4f66 | 846 | kcsan_mb(); |
3570a1bc | 847 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 848 | return arch_atomic64_fetch_inc(v); |
0ae1d994 | 849 | } |
b06ed71a | 850 | |
c020395b | 851 | static __always_inline s64 |
aa525d06 | 852 | atomic64_fetch_inc_acquire(atomic64_t *v) |
b06ed71a | 853 | { |
3570a1bc | 854 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 855 | return arch_atomic64_fetch_inc_acquire(v); |
b06ed71a DV |
856 | } |
857 | ||
c020395b | 858 | static __always_inline s64 |
aa525d06 | 859 | atomic64_fetch_inc_release(atomic64_t *v) |
b06ed71a | 860 | { |
e87c4f66 | 861 | kcsan_release(); |
3570a1bc | 862 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 863 | return arch_atomic64_fetch_inc_release(v); |
b06ed71a DV |
864 | } |
865 | ||
c020395b | 866 | static __always_inline s64 |
aa525d06 | 867 | atomic64_fetch_inc_relaxed(atomic64_t *v) |
b06ed71a | 868 | { |
3570a1bc | 869 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 870 | return arch_atomic64_fetch_inc_relaxed(v); |
b06ed71a DV |
871 | } |
872 | ||
c020395b | 873 | static __always_inline void |
aa525d06 | 874 | atomic64_dec(atomic64_t *v) |
b06ed71a | 875 | { |
3570a1bc | 876 | instrument_atomic_read_write(v, sizeof(*v)); |
b06ed71a DV |
877 | arch_atomic64_dec(v); |
878 | } | |
879 | ||
c020395b | 880 | static __always_inline s64 |
aa525d06 | 881 | atomic64_dec_return(atomic64_t *v) |
b06ed71a | 882 | { |
e87c4f66 | 883 | kcsan_mb(); |
3570a1bc | 884 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 885 | return arch_atomic64_dec_return(v); |
b06ed71a DV |
886 | } |
887 | ||
c020395b | 888 | static __always_inline s64 |
aa525d06 | 889 | atomic64_dec_return_acquire(atomic64_t *v) |
b06ed71a | 890 | { |
3570a1bc | 891 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 892 | return arch_atomic64_dec_return_acquire(v); |
b06ed71a DV |
893 | } |
894 | ||
c020395b | 895 | static __always_inline s64 |
aa525d06 | 896 | atomic64_dec_return_release(atomic64_t *v) |
b06ed71a | 897 | { |
e87c4f66 | 898 | kcsan_release(); |
3570a1bc | 899 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 900 | return arch_atomic64_dec_return_release(v); |
b06ed71a DV |
901 | } |
902 | ||
c020395b | 903 | static __always_inline s64 |
aa525d06 | 904 | atomic64_dec_return_relaxed(atomic64_t *v) |
b06ed71a | 905 | { |
3570a1bc | 906 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 907 | return arch_atomic64_dec_return_relaxed(v); |
b06ed71a DV |
908 | } |
909 | ||
c020395b | 910 | static __always_inline s64 |
aa525d06 | 911 | atomic64_fetch_dec(atomic64_t *v) |
b06ed71a | 912 | { |
e87c4f66 | 913 | kcsan_mb(); |
3570a1bc | 914 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
915 | return arch_atomic64_fetch_dec(v); |
916 | } | |
aa525d06 | 917 | |
c020395b | 918 | static __always_inline s64 |
aa525d06 MR |
919 | atomic64_fetch_dec_acquire(atomic64_t *v) |
920 | { | |
3570a1bc | 921 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
922 | return arch_atomic64_fetch_dec_acquire(v); |
923 | } | |
aa525d06 | 924 | |
c020395b | 925 | static __always_inline s64 |
aa525d06 MR |
926 | atomic64_fetch_dec_release(atomic64_t *v) |
927 | { | |
e87c4f66 | 928 | kcsan_release(); |
3570a1bc | 929 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 930 | return arch_atomic64_fetch_dec_release(v); |
b06ed71a | 931 | } |
aa525d06 | 932 | |
c020395b | 933 | static __always_inline s64 |
aa525d06 MR |
934 | atomic64_fetch_dec_relaxed(atomic64_t *v) |
935 | { | |
3570a1bc | 936 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
937 | return arch_atomic64_fetch_dec_relaxed(v); |
938 | } | |
b06ed71a | 939 | |
c020395b | 940 | static __always_inline void |
aa525d06 | 941 | atomic64_and(s64 i, atomic64_t *v) |
b06ed71a | 942 | { |
3570a1bc | 943 | instrument_atomic_read_write(v, sizeof(*v)); |
b06ed71a DV |
944 | arch_atomic64_and(i, v); |
945 | } | |
946 | ||
c020395b | 947 | static __always_inline s64 |
aa525d06 | 948 | atomic64_fetch_and(s64 i, atomic64_t *v) |
b06ed71a | 949 | { |
e87c4f66 | 950 | kcsan_mb(); |
3570a1bc | 951 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 952 | return arch_atomic64_fetch_and(i, v); |
b06ed71a DV |
953 | } |
954 | ||
c020395b | 955 | static __always_inline s64 |
aa525d06 | 956 | atomic64_fetch_and_acquire(s64 i, atomic64_t *v) |
b06ed71a | 957 | { |
3570a1bc | 958 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 959 | return arch_atomic64_fetch_and_acquire(i, v); |
b06ed71a DV |
960 | } |
961 | ||
c020395b | 962 | static __always_inline s64 |
aa525d06 | 963 | atomic64_fetch_and_release(s64 i, atomic64_t *v) |
b06ed71a | 964 | { |
e87c4f66 | 965 | kcsan_release(); |
3570a1bc | 966 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 967 | return arch_atomic64_fetch_and_release(i, v); |
b06ed71a DV |
968 | } |
969 | ||
c020395b | 970 | static __always_inline s64 |
aa525d06 | 971 | atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 972 | { |
3570a1bc | 973 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 974 | return arch_atomic64_fetch_and_relaxed(i, v); |
b06ed71a DV |
975 | } |
976 | ||
c020395b | 977 | static __always_inline void |
aa525d06 | 978 | atomic64_andnot(s64 i, atomic64_t *v) |
b06ed71a | 979 | { |
3570a1bc | 980 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 981 | arch_atomic64_andnot(i, v); |
b06ed71a DV |
982 | } |
983 | ||
c020395b | 984 | static __always_inline s64 |
aa525d06 | 985 | atomic64_fetch_andnot(s64 i, atomic64_t *v) |
b06ed71a | 986 | { |
e87c4f66 | 987 | kcsan_mb(); |
3570a1bc | 988 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 989 | return arch_atomic64_fetch_andnot(i, v); |
b06ed71a DV |
990 | } |
991 | ||
c020395b | 992 | static __always_inline s64 |
aa525d06 | 993 | atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) |
b06ed71a | 994 | { |
3570a1bc | 995 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 996 | return arch_atomic64_fetch_andnot_acquire(i, v); |
b06ed71a DV |
997 | } |
998 | ||
c020395b | 999 | static __always_inline s64 |
aa525d06 | 1000 | atomic64_fetch_andnot_release(s64 i, atomic64_t *v) |
b06ed71a | 1001 | { |
e87c4f66 | 1002 | kcsan_release(); |
3570a1bc | 1003 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1004 | return arch_atomic64_fetch_andnot_release(i, v); |
b06ed71a DV |
1005 | } |
1006 | ||
c020395b | 1007 | static __always_inline s64 |
aa525d06 | 1008 | atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 1009 | { |
3570a1bc | 1010 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1011 | return arch_atomic64_fetch_andnot_relaxed(i, v); |
b06ed71a DV |
1012 | } |
1013 | ||
c020395b | 1014 | static __always_inline void |
aa525d06 | 1015 | atomic64_or(s64 i, atomic64_t *v) |
b06ed71a | 1016 | { |
3570a1bc | 1017 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1018 | arch_atomic64_or(i, v); |
1019 | } | |
aa525d06 | 1020 | |
c020395b | 1021 | static __always_inline s64 |
aa525d06 MR |
1022 | atomic64_fetch_or(s64 i, atomic64_t *v) |
1023 | { | |
e87c4f66 | 1024 | kcsan_mb(); |
3570a1bc | 1025 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1026 | return arch_atomic64_fetch_or(i, v); |
b06ed71a DV |
1027 | } |
1028 | ||
c020395b | 1029 | static __always_inline s64 |
aa525d06 | 1030 | atomic64_fetch_or_acquire(s64 i, atomic64_t *v) |
b06ed71a | 1031 | { |
3570a1bc | 1032 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1033 | return arch_atomic64_fetch_or_acquire(i, v); |
b06ed71a DV |
1034 | } |
1035 | ||
c020395b | 1036 | static __always_inline s64 |
aa525d06 | 1037 | atomic64_fetch_or_release(s64 i, atomic64_t *v) |
b06ed71a | 1038 | { |
e87c4f66 | 1039 | kcsan_release(); |
3570a1bc | 1040 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1041 | return arch_atomic64_fetch_or_release(i, v); |
b06ed71a DV |
1042 | } |
1043 | ||
c020395b | 1044 | static __always_inline s64 |
aa525d06 | 1045 | atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 1046 | { |
3570a1bc | 1047 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1048 | return arch_atomic64_fetch_or_relaxed(i, v); |
b06ed71a DV |
1049 | } |
1050 | ||
c020395b | 1051 | static __always_inline void |
aa525d06 | 1052 | atomic64_xor(s64 i, atomic64_t *v) |
b06ed71a | 1053 | { |
3570a1bc | 1054 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1055 | arch_atomic64_xor(i, v); |
1056 | } | |
aa525d06 | 1057 | |
c020395b | 1058 | static __always_inline s64 |
aa525d06 MR |
1059 | atomic64_fetch_xor(s64 i, atomic64_t *v) |
1060 | { | |
e87c4f66 | 1061 | kcsan_mb(); |
3570a1bc | 1062 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1063 | return arch_atomic64_fetch_xor(i, v); |
b06ed71a DV |
1064 | } |
1065 | ||
c020395b | 1066 | static __always_inline s64 |
aa525d06 | 1067 | atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) |
b06ed71a | 1068 | { |
3570a1bc | 1069 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1070 | return arch_atomic64_fetch_xor_acquire(i, v); |
b06ed71a DV |
1071 | } |
1072 | ||
c020395b | 1073 | static __always_inline s64 |
aa525d06 | 1074 | atomic64_fetch_xor_release(s64 i, atomic64_t *v) |
b06ed71a | 1075 | { |
e87c4f66 | 1076 | kcsan_release(); |
3570a1bc | 1077 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1078 | return arch_atomic64_fetch_xor_release(i, v); |
b06ed71a DV |
1079 | } |
1080 | ||
c020395b | 1081 | static __always_inline s64 |
aa525d06 | 1082 | atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 1083 | { |
3570a1bc | 1084 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1085 | return arch_atomic64_fetch_xor_relaxed(i, v); |
b06ed71a DV |
1086 | } |
1087 | ||
c020395b | 1088 | static __always_inline s64 |
aa525d06 | 1089 | atomic64_xchg(atomic64_t *v, s64 i) |
b06ed71a | 1090 | { |
e87c4f66 | 1091 | kcsan_mb(); |
3570a1bc | 1092 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1093 | return arch_atomic64_xchg(v, i); |
b06ed71a DV |
1094 | } |
1095 | ||
c020395b | 1096 | static __always_inline s64 |
aa525d06 | 1097 | atomic64_xchg_acquire(atomic64_t *v, s64 i) |
b06ed71a | 1098 | { |
3570a1bc | 1099 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1100 | return arch_atomic64_xchg_acquire(v, i); |
b06ed71a DV |
1101 | } |
1102 | ||
c020395b | 1103 | static __always_inline s64 |
aa525d06 | 1104 | atomic64_xchg_release(atomic64_t *v, s64 i) |
b06ed71a | 1105 | { |
e87c4f66 | 1106 | kcsan_release(); |
3570a1bc | 1107 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1108 | return arch_atomic64_xchg_release(v, i); |
b06ed71a DV |
1109 | } |
1110 | ||
c020395b | 1111 | static __always_inline s64 |
aa525d06 | 1112 | atomic64_xchg_relaxed(atomic64_t *v, s64 i) |
b06ed71a | 1113 | { |
3570a1bc | 1114 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1115 | return arch_atomic64_xchg_relaxed(v, i); |
b06ed71a DV |
1116 | } |
1117 | ||
c020395b | 1118 | static __always_inline s64 |
aa525d06 | 1119 | atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1120 | { |
e87c4f66 | 1121 | kcsan_mb(); |
3570a1bc | 1122 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1123 | return arch_atomic64_cmpxchg(v, old, new); |
b06ed71a DV |
1124 | } |
1125 | ||
c020395b | 1126 | static __always_inline s64 |
aa525d06 | 1127 | atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1128 | { |
3570a1bc | 1129 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1130 | return arch_atomic64_cmpxchg_acquire(v, old, new); |
b06ed71a DV |
1131 | } |
1132 | ||
c020395b | 1133 | static __always_inline s64 |
aa525d06 | 1134 | atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1135 | { |
e87c4f66 | 1136 | kcsan_release(); |
3570a1bc | 1137 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1138 | return arch_atomic64_cmpxchg_release(v, old, new); |
b06ed71a DV |
1139 | } |
1140 | ||
c020395b | 1141 | static __always_inline s64 |
aa525d06 | 1142 | atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1143 | { |
3570a1bc | 1144 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1145 | return arch_atomic64_cmpxchg_relaxed(v, old, new); |
b06ed71a DV |
1146 | } |
1147 | ||
c020395b | 1148 | static __always_inline bool |
aa525d06 | 1149 | atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1150 | { |
e87c4f66 | 1151 | kcsan_mb(); |
3570a1bc ME |
1152 | instrument_atomic_read_write(v, sizeof(*v)); |
1153 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 | 1154 | return arch_atomic64_try_cmpxchg(v, old, new); |
b06ed71a DV |
1155 | } |
1156 | ||
c020395b | 1157 | static __always_inline bool |
aa525d06 | 1158 | atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1159 | { |
3570a1bc ME |
1160 | instrument_atomic_read_write(v, sizeof(*v)); |
1161 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 | 1162 | return arch_atomic64_try_cmpxchg_acquire(v, old, new); |
b06ed71a DV |
1163 | } |
1164 | ||
c020395b | 1165 | static __always_inline bool |
aa525d06 | 1166 | atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1167 | { |
e87c4f66 | 1168 | kcsan_release(); |
3570a1bc ME |
1169 | instrument_atomic_read_write(v, sizeof(*v)); |
1170 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 | 1171 | return arch_atomic64_try_cmpxchg_release(v, old, new); |
b06ed71a DV |
1172 | } |
1173 | ||
c020395b | 1174 | static __always_inline bool |
aa525d06 | 1175 | atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1176 | { |
3570a1bc ME |
1177 | instrument_atomic_read_write(v, sizeof(*v)); |
1178 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 | 1179 | return arch_atomic64_try_cmpxchg_relaxed(v, old, new); |
b06ed71a DV |
1180 | } |
1181 | ||
c020395b | 1182 | static __always_inline bool |
aa525d06 | 1183 | atomic64_sub_and_test(s64 i, atomic64_t *v) |
b06ed71a | 1184 | { |
e87c4f66 | 1185 | kcsan_mb(); |
3570a1bc | 1186 | instrument_atomic_read_write(v, sizeof(*v)); |
b06ed71a DV |
1187 | return arch_atomic64_sub_and_test(i, v); |
1188 | } | |
1189 | ||
c020395b | 1190 | static __always_inline bool |
aa525d06 | 1191 | atomic64_dec_and_test(atomic64_t *v) |
b06ed71a | 1192 | { |
e87c4f66 | 1193 | kcsan_mb(); |
3570a1bc | 1194 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1195 | return arch_atomic64_dec_and_test(v); |
b06ed71a DV |
1196 | } |
1197 | ||
c020395b | 1198 | static __always_inline bool |
aa525d06 MR |
1199 | atomic64_inc_and_test(atomic64_t *v) |
1200 | { | |
e87c4f66 | 1201 | kcsan_mb(); |
3570a1bc | 1202 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1203 | return arch_atomic64_inc_and_test(v); |
1204 | } | |
aa525d06 | 1205 | |
c020395b | 1206 | static __always_inline bool |
aa525d06 | 1207 | atomic64_add_negative(s64 i, atomic64_t *v) |
b06ed71a | 1208 | { |
e87c4f66 | 1209 | kcsan_mb(); |
3570a1bc | 1210 | instrument_atomic_read_write(v, sizeof(*v)); |
b06ed71a DV |
1211 | return arch_atomic64_add_negative(i, v); |
1212 | } | |
aa525d06 | 1213 | |
c020395b | 1214 | static __always_inline s64 |
aa525d06 MR |
1215 | atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) |
1216 | { | |
e87c4f66 | 1217 | kcsan_mb(); |
3570a1bc | 1218 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1219 | return arch_atomic64_fetch_add_unless(v, a, u); |
1220 | } | |
aa525d06 | 1221 | |
c020395b | 1222 | static __always_inline bool |
aa525d06 MR |
1223 | atomic64_add_unless(atomic64_t *v, s64 a, s64 u) |
1224 | { | |
e87c4f66 | 1225 | kcsan_mb(); |
3570a1bc | 1226 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1227 | return arch_atomic64_add_unless(v, a, u); |
1228 | } | |
aa525d06 | 1229 | |
c020395b | 1230 | static __always_inline bool |
aa525d06 MR |
1231 | atomic64_inc_not_zero(atomic64_t *v) |
1232 | { | |
e87c4f66 | 1233 | kcsan_mb(); |
3570a1bc | 1234 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1235 | return arch_atomic64_inc_not_zero(v); |
1236 | } | |
aa525d06 | 1237 | |
c020395b | 1238 | static __always_inline bool |
aa525d06 MR |
1239 | atomic64_inc_unless_negative(atomic64_t *v) |
1240 | { | |
e87c4f66 | 1241 | kcsan_mb(); |
3570a1bc | 1242 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1243 | return arch_atomic64_inc_unless_negative(v); |
1244 | } | |
b06ed71a | 1245 | |
c020395b | 1246 | static __always_inline bool |
aa525d06 MR |
1247 | atomic64_dec_unless_positive(atomic64_t *v) |
1248 | { | |
e87c4f66 | 1249 | kcsan_mb(); |
3570a1bc | 1250 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1251 | return arch_atomic64_dec_unless_positive(v); |
1252 | } | |
aa525d06 | 1253 | |
c020395b | 1254 | static __always_inline s64 |
aa525d06 MR |
1255 | atomic64_dec_if_positive(atomic64_t *v) |
1256 | { | |
e87c4f66 | 1257 | kcsan_mb(); |
3570a1bc | 1258 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1259 | return arch_atomic64_dec_if_positive(v); |
1260 | } | |
aa525d06 | 1261 | |
67d1b0de MR |
1262 | static __always_inline long |
1263 | atomic_long_read(const atomic_long_t *v) | |
1264 | { | |
1265 | instrument_atomic_read(v, sizeof(*v)); | |
1266 | return arch_atomic_long_read(v); | |
1267 | } | |
1268 | ||
1269 | static __always_inline long | |
1270 | atomic_long_read_acquire(const atomic_long_t *v) | |
1271 | { | |
1272 | instrument_atomic_read(v, sizeof(*v)); | |
1273 | return arch_atomic_long_read_acquire(v); | |
1274 | } | |
1275 | ||
1276 | static __always_inline void | |
1277 | atomic_long_set(atomic_long_t *v, long i) | |
1278 | { | |
1279 | instrument_atomic_write(v, sizeof(*v)); | |
1280 | arch_atomic_long_set(v, i); | |
1281 | } | |
1282 | ||
1283 | static __always_inline void | |
1284 | atomic_long_set_release(atomic_long_t *v, long i) | |
1285 | { | |
e87c4f66 | 1286 | kcsan_release(); |
67d1b0de MR |
1287 | instrument_atomic_write(v, sizeof(*v)); |
1288 | arch_atomic_long_set_release(v, i); | |
1289 | } | |
1290 | ||
1291 | static __always_inline void | |
1292 | atomic_long_add(long i, atomic_long_t *v) | |
1293 | { | |
1294 | instrument_atomic_read_write(v, sizeof(*v)); | |
1295 | arch_atomic_long_add(i, v); | |
1296 | } | |
1297 | ||
1298 | static __always_inline long | |
1299 | atomic_long_add_return(long i, atomic_long_t *v) | |
1300 | { | |
e87c4f66 | 1301 | kcsan_mb(); |
67d1b0de MR |
1302 | instrument_atomic_read_write(v, sizeof(*v)); |
1303 | return arch_atomic_long_add_return(i, v); | |
1304 | } | |
1305 | ||
1306 | static __always_inline long | |
1307 | atomic_long_add_return_acquire(long i, atomic_long_t *v) | |
1308 | { | |
1309 | instrument_atomic_read_write(v, sizeof(*v)); | |
1310 | return arch_atomic_long_add_return_acquire(i, v); | |
1311 | } | |
1312 | ||
1313 | static __always_inline long | |
1314 | atomic_long_add_return_release(long i, atomic_long_t *v) | |
1315 | { | |
e87c4f66 | 1316 | kcsan_release(); |
67d1b0de MR |
1317 | instrument_atomic_read_write(v, sizeof(*v)); |
1318 | return arch_atomic_long_add_return_release(i, v); | |
1319 | } | |
1320 | ||
1321 | static __always_inline long | |
1322 | atomic_long_add_return_relaxed(long i, atomic_long_t *v) | |
1323 | { | |
1324 | instrument_atomic_read_write(v, sizeof(*v)); | |
1325 | return arch_atomic_long_add_return_relaxed(i, v); | |
1326 | } | |
1327 | ||
1328 | static __always_inline long | |
1329 | atomic_long_fetch_add(long i, atomic_long_t *v) | |
1330 | { | |
e87c4f66 | 1331 | kcsan_mb(); |
67d1b0de MR |
1332 | instrument_atomic_read_write(v, sizeof(*v)); |
1333 | return arch_atomic_long_fetch_add(i, v); | |
1334 | } | |
1335 | ||
1336 | static __always_inline long | |
1337 | atomic_long_fetch_add_acquire(long i, atomic_long_t *v) | |
1338 | { | |
1339 | instrument_atomic_read_write(v, sizeof(*v)); | |
1340 | return arch_atomic_long_fetch_add_acquire(i, v); | |
1341 | } | |
1342 | ||
1343 | static __always_inline long | |
1344 | atomic_long_fetch_add_release(long i, atomic_long_t *v) | |
1345 | { | |
e87c4f66 | 1346 | kcsan_release(); |
67d1b0de MR |
1347 | instrument_atomic_read_write(v, sizeof(*v)); |
1348 | return arch_atomic_long_fetch_add_release(i, v); | |
1349 | } | |
1350 | ||
1351 | static __always_inline long | |
1352 | atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) | |
1353 | { | |
1354 | instrument_atomic_read_write(v, sizeof(*v)); | |
1355 | return arch_atomic_long_fetch_add_relaxed(i, v); | |
1356 | } | |
1357 | ||
1358 | static __always_inline void | |
1359 | atomic_long_sub(long i, atomic_long_t *v) | |
1360 | { | |
1361 | instrument_atomic_read_write(v, sizeof(*v)); | |
1362 | arch_atomic_long_sub(i, v); | |
1363 | } | |
1364 | ||
1365 | static __always_inline long | |
1366 | atomic_long_sub_return(long i, atomic_long_t *v) | |
1367 | { | |
e87c4f66 | 1368 | kcsan_mb(); |
67d1b0de MR |
1369 | instrument_atomic_read_write(v, sizeof(*v)); |
1370 | return arch_atomic_long_sub_return(i, v); | |
1371 | } | |
1372 | ||
1373 | static __always_inline long | |
1374 | atomic_long_sub_return_acquire(long i, atomic_long_t *v) | |
1375 | { | |
1376 | instrument_atomic_read_write(v, sizeof(*v)); | |
1377 | return arch_atomic_long_sub_return_acquire(i, v); | |
1378 | } | |
1379 | ||
1380 | static __always_inline long | |
1381 | atomic_long_sub_return_release(long i, atomic_long_t *v) | |
1382 | { | |
e87c4f66 | 1383 | kcsan_release(); |
67d1b0de MR |
1384 | instrument_atomic_read_write(v, sizeof(*v)); |
1385 | return arch_atomic_long_sub_return_release(i, v); | |
1386 | } | |
1387 | ||
1388 | static __always_inline long | |
1389 | atomic_long_sub_return_relaxed(long i, atomic_long_t *v) | |
1390 | { | |
1391 | instrument_atomic_read_write(v, sizeof(*v)); | |
1392 | return arch_atomic_long_sub_return_relaxed(i, v); | |
1393 | } | |
1394 | ||
1395 | static __always_inline long | |
1396 | atomic_long_fetch_sub(long i, atomic_long_t *v) | |
1397 | { | |
e87c4f66 | 1398 | kcsan_mb(); |
67d1b0de MR |
1399 | instrument_atomic_read_write(v, sizeof(*v)); |
1400 | return arch_atomic_long_fetch_sub(i, v); | |
1401 | } | |
1402 | ||
1403 | static __always_inline long | |
1404 | atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) | |
1405 | { | |
1406 | instrument_atomic_read_write(v, sizeof(*v)); | |
1407 | return arch_atomic_long_fetch_sub_acquire(i, v); | |
1408 | } | |
1409 | ||
1410 | static __always_inline long | |
1411 | atomic_long_fetch_sub_release(long i, atomic_long_t *v) | |
1412 | { | |
e87c4f66 | 1413 | kcsan_release(); |
67d1b0de MR |
1414 | instrument_atomic_read_write(v, sizeof(*v)); |
1415 | return arch_atomic_long_fetch_sub_release(i, v); | |
1416 | } | |
1417 | ||
1418 | static __always_inline long | |
1419 | atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) | |
1420 | { | |
1421 | instrument_atomic_read_write(v, sizeof(*v)); | |
1422 | return arch_atomic_long_fetch_sub_relaxed(i, v); | |
1423 | } | |
1424 | ||
1425 | static __always_inline void | |
1426 | atomic_long_inc(atomic_long_t *v) | |
1427 | { | |
1428 | instrument_atomic_read_write(v, sizeof(*v)); | |
1429 | arch_atomic_long_inc(v); | |
1430 | } | |
1431 | ||
1432 | static __always_inline long | |
1433 | atomic_long_inc_return(atomic_long_t *v) | |
1434 | { | |
e87c4f66 | 1435 | kcsan_mb(); |
67d1b0de MR |
1436 | instrument_atomic_read_write(v, sizeof(*v)); |
1437 | return arch_atomic_long_inc_return(v); | |
1438 | } | |
1439 | ||
1440 | static __always_inline long | |
1441 | atomic_long_inc_return_acquire(atomic_long_t *v) | |
1442 | { | |
1443 | instrument_atomic_read_write(v, sizeof(*v)); | |
1444 | return arch_atomic_long_inc_return_acquire(v); | |
1445 | } | |
1446 | ||
1447 | static __always_inline long | |
1448 | atomic_long_inc_return_release(atomic_long_t *v) | |
1449 | { | |
e87c4f66 | 1450 | kcsan_release(); |
67d1b0de MR |
1451 | instrument_atomic_read_write(v, sizeof(*v)); |
1452 | return arch_atomic_long_inc_return_release(v); | |
1453 | } | |
1454 | ||
1455 | static __always_inline long | |
1456 | atomic_long_inc_return_relaxed(atomic_long_t *v) | |
1457 | { | |
1458 | instrument_atomic_read_write(v, sizeof(*v)); | |
1459 | return arch_atomic_long_inc_return_relaxed(v); | |
1460 | } | |
1461 | ||
1462 | static __always_inline long | |
1463 | atomic_long_fetch_inc(atomic_long_t *v) | |
1464 | { | |
e87c4f66 | 1465 | kcsan_mb(); |
67d1b0de MR |
1466 | instrument_atomic_read_write(v, sizeof(*v)); |
1467 | return arch_atomic_long_fetch_inc(v); | |
1468 | } | |
1469 | ||
1470 | static __always_inline long | |
1471 | atomic_long_fetch_inc_acquire(atomic_long_t *v) | |
1472 | { | |
1473 | instrument_atomic_read_write(v, sizeof(*v)); | |
1474 | return arch_atomic_long_fetch_inc_acquire(v); | |
1475 | } | |
1476 | ||
1477 | static __always_inline long | |
1478 | atomic_long_fetch_inc_release(atomic_long_t *v) | |
1479 | { | |
e87c4f66 | 1480 | kcsan_release(); |
67d1b0de MR |
1481 | instrument_atomic_read_write(v, sizeof(*v)); |
1482 | return arch_atomic_long_fetch_inc_release(v); | |
1483 | } | |
1484 | ||
1485 | static __always_inline long | |
1486 | atomic_long_fetch_inc_relaxed(atomic_long_t *v) | |
1487 | { | |
1488 | instrument_atomic_read_write(v, sizeof(*v)); | |
1489 | return arch_atomic_long_fetch_inc_relaxed(v); | |
1490 | } | |
1491 | ||
1492 | static __always_inline void | |
1493 | atomic_long_dec(atomic_long_t *v) | |
1494 | { | |
1495 | instrument_atomic_read_write(v, sizeof(*v)); | |
1496 | arch_atomic_long_dec(v); | |
1497 | } | |
1498 | ||
1499 | static __always_inline long | |
1500 | atomic_long_dec_return(atomic_long_t *v) | |
1501 | { | |
e87c4f66 | 1502 | kcsan_mb(); |
67d1b0de MR |
1503 | instrument_atomic_read_write(v, sizeof(*v)); |
1504 | return arch_atomic_long_dec_return(v); | |
1505 | } | |
1506 | ||
1507 | static __always_inline long | |
1508 | atomic_long_dec_return_acquire(atomic_long_t *v) | |
1509 | { | |
1510 | instrument_atomic_read_write(v, sizeof(*v)); | |
1511 | return arch_atomic_long_dec_return_acquire(v); | |
1512 | } | |
1513 | ||
1514 | static __always_inline long | |
1515 | atomic_long_dec_return_release(atomic_long_t *v) | |
1516 | { | |
e87c4f66 | 1517 | kcsan_release(); |
67d1b0de MR |
1518 | instrument_atomic_read_write(v, sizeof(*v)); |
1519 | return arch_atomic_long_dec_return_release(v); | |
1520 | } | |
1521 | ||
1522 | static __always_inline long | |
1523 | atomic_long_dec_return_relaxed(atomic_long_t *v) | |
1524 | { | |
1525 | instrument_atomic_read_write(v, sizeof(*v)); | |
1526 | return arch_atomic_long_dec_return_relaxed(v); | |
1527 | } | |
1528 | ||
1529 | static __always_inline long | |
1530 | atomic_long_fetch_dec(atomic_long_t *v) | |
1531 | { | |
e87c4f66 | 1532 | kcsan_mb(); |
67d1b0de MR |
1533 | instrument_atomic_read_write(v, sizeof(*v)); |
1534 | return arch_atomic_long_fetch_dec(v); | |
1535 | } | |
1536 | ||
1537 | static __always_inline long | |
1538 | atomic_long_fetch_dec_acquire(atomic_long_t *v) | |
1539 | { | |
1540 | instrument_atomic_read_write(v, sizeof(*v)); | |
1541 | return arch_atomic_long_fetch_dec_acquire(v); | |
1542 | } | |
1543 | ||
1544 | static __always_inline long | |
1545 | atomic_long_fetch_dec_release(atomic_long_t *v) | |
1546 | { | |
e87c4f66 | 1547 | kcsan_release(); |
67d1b0de MR |
1548 | instrument_atomic_read_write(v, sizeof(*v)); |
1549 | return arch_atomic_long_fetch_dec_release(v); | |
1550 | } | |
1551 | ||
1552 | static __always_inline long | |
1553 | atomic_long_fetch_dec_relaxed(atomic_long_t *v) | |
1554 | { | |
1555 | instrument_atomic_read_write(v, sizeof(*v)); | |
1556 | return arch_atomic_long_fetch_dec_relaxed(v); | |
1557 | } | |
1558 | ||
1559 | static __always_inline void | |
1560 | atomic_long_and(long i, atomic_long_t *v) | |
1561 | { | |
1562 | instrument_atomic_read_write(v, sizeof(*v)); | |
1563 | arch_atomic_long_and(i, v); | |
1564 | } | |
1565 | ||
1566 | static __always_inline long | |
1567 | atomic_long_fetch_and(long i, atomic_long_t *v) | |
1568 | { | |
e87c4f66 | 1569 | kcsan_mb(); |
67d1b0de MR |
1570 | instrument_atomic_read_write(v, sizeof(*v)); |
1571 | return arch_atomic_long_fetch_and(i, v); | |
1572 | } | |
1573 | ||
1574 | static __always_inline long | |
1575 | atomic_long_fetch_and_acquire(long i, atomic_long_t *v) | |
1576 | { | |
1577 | instrument_atomic_read_write(v, sizeof(*v)); | |
1578 | return arch_atomic_long_fetch_and_acquire(i, v); | |
1579 | } | |
1580 | ||
1581 | static __always_inline long | |
1582 | atomic_long_fetch_and_release(long i, atomic_long_t *v) | |
1583 | { | |
e87c4f66 | 1584 | kcsan_release(); |
67d1b0de MR |
1585 | instrument_atomic_read_write(v, sizeof(*v)); |
1586 | return arch_atomic_long_fetch_and_release(i, v); | |
1587 | } | |
1588 | ||
1589 | static __always_inline long | |
1590 | atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) | |
1591 | { | |
1592 | instrument_atomic_read_write(v, sizeof(*v)); | |
1593 | return arch_atomic_long_fetch_and_relaxed(i, v); | |
1594 | } | |
1595 | ||
1596 | static __always_inline void | |
1597 | atomic_long_andnot(long i, atomic_long_t *v) | |
1598 | { | |
1599 | instrument_atomic_read_write(v, sizeof(*v)); | |
1600 | arch_atomic_long_andnot(i, v); | |
1601 | } | |
1602 | ||
1603 | static __always_inline long | |
1604 | atomic_long_fetch_andnot(long i, atomic_long_t *v) | |
1605 | { | |
e87c4f66 | 1606 | kcsan_mb(); |
67d1b0de MR |
1607 | instrument_atomic_read_write(v, sizeof(*v)); |
1608 | return arch_atomic_long_fetch_andnot(i, v); | |
1609 | } | |
1610 | ||
1611 | static __always_inline long | |
1612 | atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) | |
1613 | { | |
1614 | instrument_atomic_read_write(v, sizeof(*v)); | |
1615 | return arch_atomic_long_fetch_andnot_acquire(i, v); | |
1616 | } | |
1617 | ||
1618 | static __always_inline long | |
1619 | atomic_long_fetch_andnot_release(long i, atomic_long_t *v) | |
1620 | { | |
e87c4f66 | 1621 | kcsan_release(); |
67d1b0de MR |
1622 | instrument_atomic_read_write(v, sizeof(*v)); |
1623 | return arch_atomic_long_fetch_andnot_release(i, v); | |
1624 | } | |
1625 | ||
1626 | static __always_inline long | |
1627 | atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) | |
1628 | { | |
1629 | instrument_atomic_read_write(v, sizeof(*v)); | |
1630 | return arch_atomic_long_fetch_andnot_relaxed(i, v); | |
1631 | } | |
1632 | ||
1633 | static __always_inline void | |
1634 | atomic_long_or(long i, atomic_long_t *v) | |
1635 | { | |
1636 | instrument_atomic_read_write(v, sizeof(*v)); | |
1637 | arch_atomic_long_or(i, v); | |
1638 | } | |
1639 | ||
1640 | static __always_inline long | |
1641 | atomic_long_fetch_or(long i, atomic_long_t *v) | |
1642 | { | |
e87c4f66 | 1643 | kcsan_mb(); |
67d1b0de MR |
1644 | instrument_atomic_read_write(v, sizeof(*v)); |
1645 | return arch_atomic_long_fetch_or(i, v); | |
1646 | } | |
1647 | ||
1648 | static __always_inline long | |
1649 | atomic_long_fetch_or_acquire(long i, atomic_long_t *v) | |
1650 | { | |
1651 | instrument_atomic_read_write(v, sizeof(*v)); | |
1652 | return arch_atomic_long_fetch_or_acquire(i, v); | |
1653 | } | |
1654 | ||
1655 | static __always_inline long | |
1656 | atomic_long_fetch_or_release(long i, atomic_long_t *v) | |
1657 | { | |
e87c4f66 | 1658 | kcsan_release(); |
67d1b0de MR |
1659 | instrument_atomic_read_write(v, sizeof(*v)); |
1660 | return arch_atomic_long_fetch_or_release(i, v); | |
1661 | } | |
1662 | ||
1663 | static __always_inline long | |
1664 | atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) | |
1665 | { | |
1666 | instrument_atomic_read_write(v, sizeof(*v)); | |
1667 | return arch_atomic_long_fetch_or_relaxed(i, v); | |
1668 | } | |
1669 | ||
1670 | static __always_inline void | |
1671 | atomic_long_xor(long i, atomic_long_t *v) | |
1672 | { | |
1673 | instrument_atomic_read_write(v, sizeof(*v)); | |
1674 | arch_atomic_long_xor(i, v); | |
1675 | } | |
1676 | ||
1677 | static __always_inline long | |
1678 | atomic_long_fetch_xor(long i, atomic_long_t *v) | |
1679 | { | |
e87c4f66 | 1680 | kcsan_mb(); |
67d1b0de MR |
1681 | instrument_atomic_read_write(v, sizeof(*v)); |
1682 | return arch_atomic_long_fetch_xor(i, v); | |
1683 | } | |
1684 | ||
1685 | static __always_inline long | |
1686 | atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) | |
1687 | { | |
1688 | instrument_atomic_read_write(v, sizeof(*v)); | |
1689 | return arch_atomic_long_fetch_xor_acquire(i, v); | |
1690 | } | |
1691 | ||
1692 | static __always_inline long | |
1693 | atomic_long_fetch_xor_release(long i, atomic_long_t *v) | |
1694 | { | |
e87c4f66 | 1695 | kcsan_release(); |
67d1b0de MR |
1696 | instrument_atomic_read_write(v, sizeof(*v)); |
1697 | return arch_atomic_long_fetch_xor_release(i, v); | |
1698 | } | |
1699 | ||
1700 | static __always_inline long | |
1701 | atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) | |
1702 | { | |
1703 | instrument_atomic_read_write(v, sizeof(*v)); | |
1704 | return arch_atomic_long_fetch_xor_relaxed(i, v); | |
1705 | } | |
1706 | ||
1707 | static __always_inline long | |
1708 | atomic_long_xchg(atomic_long_t *v, long i) | |
1709 | { | |
e87c4f66 | 1710 | kcsan_mb(); |
67d1b0de MR |
1711 | instrument_atomic_read_write(v, sizeof(*v)); |
1712 | return arch_atomic_long_xchg(v, i); | |
1713 | } | |
1714 | ||
1715 | static __always_inline long | |
1716 | atomic_long_xchg_acquire(atomic_long_t *v, long i) | |
1717 | { | |
1718 | instrument_atomic_read_write(v, sizeof(*v)); | |
1719 | return arch_atomic_long_xchg_acquire(v, i); | |
1720 | } | |
1721 | ||
1722 | static __always_inline long | |
1723 | atomic_long_xchg_release(atomic_long_t *v, long i) | |
1724 | { | |
e87c4f66 | 1725 | kcsan_release(); |
67d1b0de MR |
1726 | instrument_atomic_read_write(v, sizeof(*v)); |
1727 | return arch_atomic_long_xchg_release(v, i); | |
1728 | } | |
1729 | ||
1730 | static __always_inline long | |
1731 | atomic_long_xchg_relaxed(atomic_long_t *v, long i) | |
1732 | { | |
1733 | instrument_atomic_read_write(v, sizeof(*v)); | |
1734 | return arch_atomic_long_xchg_relaxed(v, i); | |
1735 | } | |
1736 | ||
1737 | static __always_inline long | |
1738 | atomic_long_cmpxchg(atomic_long_t *v, long old, long new) | |
1739 | { | |
e87c4f66 | 1740 | kcsan_mb(); |
67d1b0de MR |
1741 | instrument_atomic_read_write(v, sizeof(*v)); |
1742 | return arch_atomic_long_cmpxchg(v, old, new); | |
1743 | } | |
1744 | ||
1745 | static __always_inline long | |
1746 | atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) | |
1747 | { | |
1748 | instrument_atomic_read_write(v, sizeof(*v)); | |
1749 | return arch_atomic_long_cmpxchg_acquire(v, old, new); | |
1750 | } | |
1751 | ||
1752 | static __always_inline long | |
1753 | atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) | |
1754 | { | |
e87c4f66 | 1755 | kcsan_release(); |
67d1b0de MR |
1756 | instrument_atomic_read_write(v, sizeof(*v)); |
1757 | return arch_atomic_long_cmpxchg_release(v, old, new); | |
1758 | } | |
1759 | ||
1760 | static __always_inline long | |
1761 | atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) | |
1762 | { | |
1763 | instrument_atomic_read_write(v, sizeof(*v)); | |
1764 | return arch_atomic_long_cmpxchg_relaxed(v, old, new); | |
1765 | } | |
1766 | ||
1767 | static __always_inline bool | |
1768 | atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) | |
1769 | { | |
e87c4f66 | 1770 | kcsan_mb(); |
67d1b0de MR |
1771 | instrument_atomic_read_write(v, sizeof(*v)); |
1772 | instrument_atomic_read_write(old, sizeof(*old)); | |
1773 | return arch_atomic_long_try_cmpxchg(v, old, new); | |
1774 | } | |
1775 | ||
1776 | static __always_inline bool | |
1777 | atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) | |
1778 | { | |
1779 | instrument_atomic_read_write(v, sizeof(*v)); | |
1780 | instrument_atomic_read_write(old, sizeof(*old)); | |
1781 | return arch_atomic_long_try_cmpxchg_acquire(v, old, new); | |
1782 | } | |
1783 | ||
1784 | static __always_inline bool | |
1785 | atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) | |
1786 | { | |
e87c4f66 | 1787 | kcsan_release(); |
67d1b0de MR |
1788 | instrument_atomic_read_write(v, sizeof(*v)); |
1789 | instrument_atomic_read_write(old, sizeof(*old)); | |
1790 | return arch_atomic_long_try_cmpxchg_release(v, old, new); | |
1791 | } | |
1792 | ||
1793 | static __always_inline bool | |
1794 | atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) | |
1795 | { | |
1796 | instrument_atomic_read_write(v, sizeof(*v)); | |
1797 | instrument_atomic_read_write(old, sizeof(*old)); | |
1798 | return arch_atomic_long_try_cmpxchg_relaxed(v, old, new); | |
1799 | } | |
1800 | ||
1801 | static __always_inline bool | |
1802 | atomic_long_sub_and_test(long i, atomic_long_t *v) | |
1803 | { | |
e87c4f66 | 1804 | kcsan_mb(); |
67d1b0de MR |
1805 | instrument_atomic_read_write(v, sizeof(*v)); |
1806 | return arch_atomic_long_sub_and_test(i, v); | |
1807 | } | |
1808 | ||
1809 | static __always_inline bool | |
1810 | atomic_long_dec_and_test(atomic_long_t *v) | |
1811 | { | |
e87c4f66 | 1812 | kcsan_mb(); |
67d1b0de MR |
1813 | instrument_atomic_read_write(v, sizeof(*v)); |
1814 | return arch_atomic_long_dec_and_test(v); | |
1815 | } | |
1816 | ||
1817 | static __always_inline bool | |
1818 | atomic_long_inc_and_test(atomic_long_t *v) | |
1819 | { | |
e87c4f66 | 1820 | kcsan_mb(); |
67d1b0de MR |
1821 | instrument_atomic_read_write(v, sizeof(*v)); |
1822 | return arch_atomic_long_inc_and_test(v); | |
1823 | } | |
1824 | ||
1825 | static __always_inline bool | |
1826 | atomic_long_add_negative(long i, atomic_long_t *v) | |
1827 | { | |
e87c4f66 | 1828 | kcsan_mb(); |
67d1b0de MR |
1829 | instrument_atomic_read_write(v, sizeof(*v)); |
1830 | return arch_atomic_long_add_negative(i, v); | |
1831 | } | |
1832 | ||
1833 | static __always_inline long | |
1834 | atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) | |
1835 | { | |
e87c4f66 | 1836 | kcsan_mb(); |
67d1b0de MR |
1837 | instrument_atomic_read_write(v, sizeof(*v)); |
1838 | return arch_atomic_long_fetch_add_unless(v, a, u); | |
1839 | } | |
1840 | ||
1841 | static __always_inline bool | |
1842 | atomic_long_add_unless(atomic_long_t *v, long a, long u) | |
1843 | { | |
e87c4f66 | 1844 | kcsan_mb(); |
67d1b0de MR |
1845 | instrument_atomic_read_write(v, sizeof(*v)); |
1846 | return arch_atomic_long_add_unless(v, a, u); | |
1847 | } | |
1848 | ||
1849 | static __always_inline bool | |
1850 | atomic_long_inc_not_zero(atomic_long_t *v) | |
1851 | { | |
e87c4f66 | 1852 | kcsan_mb(); |
67d1b0de MR |
1853 | instrument_atomic_read_write(v, sizeof(*v)); |
1854 | return arch_atomic_long_inc_not_zero(v); | |
1855 | } | |
1856 | ||
1857 | static __always_inline bool | |
1858 | atomic_long_inc_unless_negative(atomic_long_t *v) | |
1859 | { | |
e87c4f66 | 1860 | kcsan_mb(); |
67d1b0de MR |
1861 | instrument_atomic_read_write(v, sizeof(*v)); |
1862 | return arch_atomic_long_inc_unless_negative(v); | |
1863 | } | |
1864 | ||
1865 | static __always_inline bool | |
1866 | atomic_long_dec_unless_positive(atomic_long_t *v) | |
1867 | { | |
e87c4f66 | 1868 | kcsan_mb(); |
67d1b0de MR |
1869 | instrument_atomic_read_write(v, sizeof(*v)); |
1870 | return arch_atomic_long_dec_unless_positive(v); | |
1871 | } | |
1872 | ||
1873 | static __always_inline long | |
1874 | atomic_long_dec_if_positive(atomic_long_t *v) | |
1875 | { | |
e87c4f66 | 1876 | kcsan_mb(); |
67d1b0de MR |
1877 | instrument_atomic_read_write(v, sizeof(*v)); |
1878 | return arch_atomic_long_dec_if_positive(v); | |
1879 | } | |
1880 | ||
29f006fd PZ |
1881 | #define xchg(ptr, ...) \ |
1882 | ({ \ | |
1883 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 1884 | kcsan_mb(); \ |
29f006fd PZ |
1885 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
1886 | arch_xchg(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1887 | }) |
aa525d06 | 1888 | |
29f006fd PZ |
1889 | #define xchg_acquire(ptr, ...) \ |
1890 | ({ \ | |
1891 | typeof(ptr) __ai_ptr = (ptr); \ | |
1892 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1893 | arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1894 | }) |
aa525d06 | 1895 | |
29f006fd PZ |
1896 | #define xchg_release(ptr, ...) \ |
1897 | ({ \ | |
1898 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 1899 | kcsan_release(); \ |
29f006fd PZ |
1900 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
1901 | arch_xchg_release(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1902 | }) |
aa525d06 | 1903 | |
29f006fd PZ |
1904 | #define xchg_relaxed(ptr, ...) \ |
1905 | ({ \ | |
1906 | typeof(ptr) __ai_ptr = (ptr); \ | |
1907 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1908 | arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1909 | }) |
aa525d06 | 1910 | |
29f006fd PZ |
1911 | #define cmpxchg(ptr, ...) \ |
1912 | ({ \ | |
1913 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 1914 | kcsan_mb(); \ |
29f006fd PZ |
1915 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
1916 | arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ | |
f9881cc4 | 1917 | }) |
aa525d06 | 1918 | |
29f006fd PZ |
1919 | #define cmpxchg_acquire(ptr, ...) \ |
1920 | ({ \ | |
1921 | typeof(ptr) __ai_ptr = (ptr); \ | |
1922 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1923 | arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1924 | }) |
aa525d06 | 1925 | |
29f006fd PZ |
1926 | #define cmpxchg_release(ptr, ...) \ |
1927 | ({ \ | |
1928 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 1929 | kcsan_release(); \ |
29f006fd PZ |
1930 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
1931 | arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1932 | }) |
aa525d06 | 1933 | |
29f006fd PZ |
1934 | #define cmpxchg_relaxed(ptr, ...) \ |
1935 | ({ \ | |
1936 | typeof(ptr) __ai_ptr = (ptr); \ | |
1937 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1938 | arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1939 | }) |
f9881cc4 | 1940 | |
29f006fd PZ |
1941 | #define cmpxchg64(ptr, ...) \ |
1942 | ({ \ | |
1943 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 1944 | kcsan_mb(); \ |
29f006fd PZ |
1945 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
1946 | arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a | 1947 | }) |
aa525d06 | 1948 | |
29f006fd PZ |
1949 | #define cmpxchg64_acquire(ptr, ...) \ |
1950 | ({ \ | |
1951 | typeof(ptr) __ai_ptr = (ptr); \ | |
1952 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1953 | arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1954 | }) |
aa525d06 | 1955 | |
29f006fd PZ |
1956 | #define cmpxchg64_release(ptr, ...) \ |
1957 | ({ \ | |
1958 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 1959 | kcsan_release(); \ |
29f006fd PZ |
1960 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
1961 | arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ | |
aa525d06 | 1962 | }) |
b06ed71a | 1963 | |
29f006fd PZ |
1964 | #define cmpxchg64_relaxed(ptr, ...) \ |
1965 | ({ \ | |
1966 | typeof(ptr) __ai_ptr = (ptr); \ | |
1967 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1968 | arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a DV |
1969 | }) |
1970 | ||
29f006fd PZ |
1971 | #define try_cmpxchg(ptr, oldp, ...) \ |
1972 | ({ \ | |
1973 | typeof(ptr) __ai_ptr = (ptr); \ | |
1974 | typeof(oldp) __ai_oldp = (oldp); \ | |
e87c4f66 | 1975 | kcsan_mb(); \ |
29f006fd PZ |
1976 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
1977 | instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
1978 | arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ | |
1979 | }) | |
29f006fd | 1980 | |
29f006fd PZ |
1981 | #define try_cmpxchg_acquire(ptr, oldp, ...) \ |
1982 | ({ \ | |
1983 | typeof(ptr) __ai_ptr = (ptr); \ | |
1984 | typeof(oldp) __ai_oldp = (oldp); \ | |
1985 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
1986 | instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
1987 | arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ | |
1988 | }) | |
29f006fd | 1989 | |
29f006fd PZ |
1990 | #define try_cmpxchg_release(ptr, oldp, ...) \ |
1991 | ({ \ | |
1992 | typeof(ptr) __ai_ptr = (ptr); \ | |
1993 | typeof(oldp) __ai_oldp = (oldp); \ | |
e87c4f66 | 1994 | kcsan_release(); \ |
29f006fd PZ |
1995 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
1996 | instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
1997 | arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ | |
1998 | }) | |
29f006fd | 1999 | |
29f006fd PZ |
2000 | #define try_cmpxchg_relaxed(ptr, oldp, ...) \ |
2001 | ({ \ | |
2002 | typeof(ptr) __ai_ptr = (ptr); \ | |
2003 | typeof(oldp) __ai_oldp = (oldp); \ | |
2004 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
2005 | instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
2006 | arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ | |
2007 | }) | |
29f006fd | 2008 | |
0aa7be05 UB |
2009 | #define try_cmpxchg64(ptr, oldp, ...) \ |
2010 | ({ \ | |
2011 | typeof(ptr) __ai_ptr = (ptr); \ | |
2012 | typeof(oldp) __ai_oldp = (oldp); \ | |
2013 | kcsan_mb(); \ | |
2014 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
2015 | instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
2016 | arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \ | |
2017 | }) | |
2018 | ||
2019 | #define try_cmpxchg64_acquire(ptr, oldp, ...) \ | |
2020 | ({ \ | |
2021 | typeof(ptr) __ai_ptr = (ptr); \ | |
2022 | typeof(oldp) __ai_oldp = (oldp); \ | |
2023 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
2024 | instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
2025 | arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ | |
2026 | }) | |
2027 | ||
2028 | #define try_cmpxchg64_release(ptr, oldp, ...) \ | |
2029 | ({ \ | |
2030 | typeof(ptr) __ai_ptr = (ptr); \ | |
2031 | typeof(oldp) __ai_oldp = (oldp); \ | |
2032 | kcsan_release(); \ | |
2033 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
2034 | instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
2035 | arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ | |
2036 | }) | |
2037 | ||
2038 | #define try_cmpxchg64_relaxed(ptr, oldp, ...) \ | |
2039 | ({ \ | |
2040 | typeof(ptr) __ai_ptr = (ptr); \ | |
2041 | typeof(oldp) __ai_oldp = (oldp); \ | |
2042 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
2043 | instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
2044 | arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ | |
2045 | }) | |
2046 | ||
29f006fd PZ |
2047 | #define cmpxchg_local(ptr, ...) \ |
2048 | ({ \ | |
2049 | typeof(ptr) __ai_ptr = (ptr); \ | |
2050 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
2051 | arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a DV |
2052 | }) |
2053 | ||
29f006fd PZ |
2054 | #define cmpxchg64_local(ptr, ...) \ |
2055 | ({ \ | |
2056 | typeof(ptr) __ai_ptr = (ptr); \ | |
2057 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ | |
2058 | arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a DV |
2059 | }) |
2060 | ||
29f006fd PZ |
2061 | #define sync_cmpxchg(ptr, ...) \ |
2062 | ({ \ | |
2063 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 2064 | kcsan_mb(); \ |
29f006fd PZ |
2065 | instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
2066 | arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a DV |
2067 | }) |
2068 | ||
29f006fd PZ |
2069 | #define cmpxchg_double(ptr, ...) \ |
2070 | ({ \ | |
2071 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 2072 | kcsan_mb(); \ |
29f006fd PZ |
2073 | instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ |
2074 | arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a DV |
2075 | }) |
2076 | ||
aa525d06 | 2077 | |
29f006fd PZ |
2078 | #define cmpxchg_double_local(ptr, ...) \ |
2079 | ({ \ | |
2080 | typeof(ptr) __ai_ptr = (ptr); \ | |
2081 | instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ | |
2082 | arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ | |
b06ed71a DV |
2083 | }) |
2084 | ||
e3d18cee | 2085 | #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ |
0aa7be05 | 2086 | // 764f741eb77a7ad565dc8d99ce2837d5542e8aee |