Commit | Line | Data |
---|---|---|
aa525d06 MR |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | // Generated by scripts/atomic/gen-atomic-instrumented.sh | |
4 | // DO NOT MODIFY THIS FILE DIRECTLY | |
5 | ||
ac605bee DV |
6 | /* |
7 | * This file provides wrappers with KASAN instrumentation for atomic operations. | |
8 | * To use this functionality an arch's atomic.h file needs to define all | |
9 | * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include | |
10 | * this file at the end. This file provides atomic_read() that forwards to | |
11 | * arch_atomic_read() for actual atomic operation. | |
12 | * Note: if an arch atomic operation is implemented by means of other atomic | |
13 | * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use | |
14 | * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid | |
15 | * double instrumentation. | |
16 | */ | |
e3d18cee MR |
17 | #ifndef _LINUX_ATOMIC_INSTRUMENTED_H |
18 | #define _LINUX_ATOMIC_INSTRUMENTED_H | |
aa525d06 MR |
19 | |
20 | #include <linux/build_bug.h> | |
c020395b | 21 | #include <linux/compiler.h> |
ed8af2e4 | 22 | #include <linux/instrumented.h> |
aa525d06 | 23 | |
c020395b | 24 | static __always_inline int |
aa525d06 MR |
25 | atomic_read(const atomic_t *v) |
26 | { | |
ed8af2e4 | 27 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
28 | return arch_atomic_read(v); |
29 | } | |
aa525d06 | 30 | |
c020395b | 31 | static __always_inline int |
aa525d06 MR |
32 | atomic_read_acquire(const atomic_t *v) |
33 | { | |
ed8af2e4 | 34 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
35 | return arch_atomic_read_acquire(v); |
36 | } | |
aa525d06 | 37 | |
c020395b | 38 | static __always_inline void |
aa525d06 MR |
39 | atomic_set(atomic_t *v, int i) |
40 | { | |
ed8af2e4 | 41 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
42 | arch_atomic_set(v, i); |
43 | } | |
aa525d06 | 44 | |
c020395b | 45 | static __always_inline void |
aa525d06 MR |
46 | atomic_set_release(atomic_t *v, int i) |
47 | { | |
e87c4f66 | 48 | kcsan_release(); |
ed8af2e4 | 49 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
50 | arch_atomic_set_release(v, i); |
51 | } | |
aa525d06 | 52 | |
c020395b | 53 | static __always_inline void |
aa525d06 MR |
54 | atomic_add(int i, atomic_t *v) |
55 | { | |
3570a1bc | 56 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
57 | arch_atomic_add(i, v); |
58 | } | |
aa525d06 | 59 | |
c020395b | 60 | static __always_inline int |
aa525d06 MR |
61 | atomic_add_return(int i, atomic_t *v) |
62 | { | |
e87c4f66 | 63 | kcsan_mb(); |
3570a1bc | 64 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
65 | return arch_atomic_add_return(i, v); |
66 | } | |
aa525d06 | 67 | |
c020395b | 68 | static __always_inline int |
aa525d06 MR |
69 | atomic_add_return_acquire(int i, atomic_t *v) |
70 | { | |
3570a1bc | 71 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
72 | return arch_atomic_add_return_acquire(i, v); |
73 | } | |
aa525d06 | 74 | |
c020395b | 75 | static __always_inline int |
aa525d06 MR |
76 | atomic_add_return_release(int i, atomic_t *v) |
77 | { | |
e87c4f66 | 78 | kcsan_release(); |
3570a1bc | 79 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
80 | return arch_atomic_add_return_release(i, v); |
81 | } | |
aa525d06 | 82 | |
c020395b | 83 | static __always_inline int |
aa525d06 MR |
84 | atomic_add_return_relaxed(int i, atomic_t *v) |
85 | { | |
3570a1bc | 86 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
87 | return arch_atomic_add_return_relaxed(i, v); |
88 | } | |
aa525d06 | 89 | |
c020395b | 90 | static __always_inline int |
aa525d06 MR |
91 | atomic_fetch_add(int i, atomic_t *v) |
92 | { | |
e87c4f66 | 93 | kcsan_mb(); |
3570a1bc | 94 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
95 | return arch_atomic_fetch_add(i, v); |
96 | } | |
aa525d06 | 97 | |
c020395b | 98 | static __always_inline int |
aa525d06 MR |
99 | atomic_fetch_add_acquire(int i, atomic_t *v) |
100 | { | |
3570a1bc | 101 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
102 | return arch_atomic_fetch_add_acquire(i, v); |
103 | } | |
aa525d06 | 104 | |
c020395b | 105 | static __always_inline int |
aa525d06 MR |
106 | atomic_fetch_add_release(int i, atomic_t *v) |
107 | { | |
e87c4f66 | 108 | kcsan_release(); |
3570a1bc | 109 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
110 | return arch_atomic_fetch_add_release(i, v); |
111 | } | |
aa525d06 | 112 | |
c020395b | 113 | static __always_inline int |
aa525d06 MR |
114 | atomic_fetch_add_relaxed(int i, atomic_t *v) |
115 | { | |
3570a1bc | 116 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
117 | return arch_atomic_fetch_add_relaxed(i, v); |
118 | } | |
aa525d06 | 119 | |
c020395b | 120 | static __always_inline void |
aa525d06 MR |
121 | atomic_sub(int i, atomic_t *v) |
122 | { | |
3570a1bc | 123 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
124 | arch_atomic_sub(i, v); |
125 | } | |
aa525d06 | 126 | |
c020395b | 127 | static __always_inline int |
aa525d06 MR |
128 | atomic_sub_return(int i, atomic_t *v) |
129 | { | |
e87c4f66 | 130 | kcsan_mb(); |
3570a1bc | 131 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
132 | return arch_atomic_sub_return(i, v); |
133 | } | |
aa525d06 | 134 | |
c020395b | 135 | static __always_inline int |
aa525d06 MR |
136 | atomic_sub_return_acquire(int i, atomic_t *v) |
137 | { | |
3570a1bc | 138 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
139 | return arch_atomic_sub_return_acquire(i, v); |
140 | } | |
aa525d06 | 141 | |
c020395b | 142 | static __always_inline int |
aa525d06 MR |
143 | atomic_sub_return_release(int i, atomic_t *v) |
144 | { | |
e87c4f66 | 145 | kcsan_release(); |
3570a1bc | 146 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
147 | return arch_atomic_sub_return_release(i, v); |
148 | } | |
aa525d06 | 149 | |
c020395b | 150 | static __always_inline int |
aa525d06 MR |
151 | atomic_sub_return_relaxed(int i, atomic_t *v) |
152 | { | |
3570a1bc | 153 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
154 | return arch_atomic_sub_return_relaxed(i, v); |
155 | } | |
aa525d06 | 156 | |
c020395b | 157 | static __always_inline int |
aa525d06 MR |
158 | atomic_fetch_sub(int i, atomic_t *v) |
159 | { | |
e87c4f66 | 160 | kcsan_mb(); |
3570a1bc | 161 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
162 | return arch_atomic_fetch_sub(i, v); |
163 | } | |
aa525d06 | 164 | |
c020395b | 165 | static __always_inline int |
aa525d06 MR |
166 | atomic_fetch_sub_acquire(int i, atomic_t *v) |
167 | { | |
3570a1bc | 168 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
169 | return arch_atomic_fetch_sub_acquire(i, v); |
170 | } | |
aa525d06 | 171 | |
c020395b | 172 | static __always_inline int |
aa525d06 MR |
173 | atomic_fetch_sub_release(int i, atomic_t *v) |
174 | { | |
e87c4f66 | 175 | kcsan_release(); |
3570a1bc | 176 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
177 | return arch_atomic_fetch_sub_release(i, v); |
178 | } | |
aa525d06 | 179 | |
c020395b | 180 | static __always_inline int |
aa525d06 MR |
181 | atomic_fetch_sub_relaxed(int i, atomic_t *v) |
182 | { | |
3570a1bc | 183 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
184 | return arch_atomic_fetch_sub_relaxed(i, v); |
185 | } | |
aa525d06 | 186 | |
c020395b | 187 | static __always_inline void |
aa525d06 MR |
188 | atomic_inc(atomic_t *v) |
189 | { | |
3570a1bc | 190 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
191 | arch_atomic_inc(v); |
192 | } | |
aa525d06 | 193 | |
c020395b | 194 | static __always_inline int |
aa525d06 MR |
195 | atomic_inc_return(atomic_t *v) |
196 | { | |
e87c4f66 | 197 | kcsan_mb(); |
3570a1bc | 198 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
199 | return arch_atomic_inc_return(v); |
200 | } | |
aa525d06 | 201 | |
c020395b | 202 | static __always_inline int |
aa525d06 MR |
203 | atomic_inc_return_acquire(atomic_t *v) |
204 | { | |
3570a1bc | 205 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
206 | return arch_atomic_inc_return_acquire(v); |
207 | } | |
aa525d06 | 208 | |
c020395b | 209 | static __always_inline int |
aa525d06 MR |
210 | atomic_inc_return_release(atomic_t *v) |
211 | { | |
e87c4f66 | 212 | kcsan_release(); |
3570a1bc | 213 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
214 | return arch_atomic_inc_return_release(v); |
215 | } | |
aa525d06 | 216 | |
c020395b | 217 | static __always_inline int |
aa525d06 MR |
218 | atomic_inc_return_relaxed(atomic_t *v) |
219 | { | |
3570a1bc | 220 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
221 | return arch_atomic_inc_return_relaxed(v); |
222 | } | |
aa525d06 | 223 | |
c020395b | 224 | static __always_inline int |
aa525d06 MR |
225 | atomic_fetch_inc(atomic_t *v) |
226 | { | |
e87c4f66 | 227 | kcsan_mb(); |
3570a1bc | 228 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
229 | return arch_atomic_fetch_inc(v); |
230 | } | |
aa525d06 | 231 | |
c020395b | 232 | static __always_inline int |
aa525d06 MR |
233 | atomic_fetch_inc_acquire(atomic_t *v) |
234 | { | |
3570a1bc | 235 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
236 | return arch_atomic_fetch_inc_acquire(v); |
237 | } | |
aa525d06 | 238 | |
c020395b | 239 | static __always_inline int |
aa525d06 MR |
240 | atomic_fetch_inc_release(atomic_t *v) |
241 | { | |
e87c4f66 | 242 | kcsan_release(); |
3570a1bc | 243 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
244 | return arch_atomic_fetch_inc_release(v); |
245 | } | |
aa525d06 | 246 | |
c020395b | 247 | static __always_inline int |
aa525d06 MR |
248 | atomic_fetch_inc_relaxed(atomic_t *v) |
249 | { | |
3570a1bc | 250 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
251 | return arch_atomic_fetch_inc_relaxed(v); |
252 | } | |
aa525d06 | 253 | |
c020395b | 254 | static __always_inline void |
aa525d06 MR |
255 | atomic_dec(atomic_t *v) |
256 | { | |
3570a1bc | 257 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
258 | arch_atomic_dec(v); |
259 | } | |
aa525d06 | 260 | |
c020395b | 261 | static __always_inline int |
aa525d06 MR |
262 | atomic_dec_return(atomic_t *v) |
263 | { | |
e87c4f66 | 264 | kcsan_mb(); |
3570a1bc | 265 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
266 | return arch_atomic_dec_return(v); |
267 | } | |
aa525d06 | 268 | |
c020395b | 269 | static __always_inline int |
aa525d06 MR |
270 | atomic_dec_return_acquire(atomic_t *v) |
271 | { | |
3570a1bc | 272 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
273 | return arch_atomic_dec_return_acquire(v); |
274 | } | |
aa525d06 | 275 | |
c020395b | 276 | static __always_inline int |
aa525d06 MR |
277 | atomic_dec_return_release(atomic_t *v) |
278 | { | |
e87c4f66 | 279 | kcsan_release(); |
3570a1bc | 280 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
281 | return arch_atomic_dec_return_release(v); |
282 | } | |
aa525d06 | 283 | |
c020395b | 284 | static __always_inline int |
aa525d06 MR |
285 | atomic_dec_return_relaxed(atomic_t *v) |
286 | { | |
3570a1bc | 287 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
288 | return arch_atomic_dec_return_relaxed(v); |
289 | } | |
aa525d06 | 290 | |
c020395b | 291 | static __always_inline int |
aa525d06 MR |
292 | atomic_fetch_dec(atomic_t *v) |
293 | { | |
e87c4f66 | 294 | kcsan_mb(); |
3570a1bc | 295 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
296 | return arch_atomic_fetch_dec(v); |
297 | } | |
aa525d06 | 298 | |
c020395b | 299 | static __always_inline int |
aa525d06 MR |
300 | atomic_fetch_dec_acquire(atomic_t *v) |
301 | { | |
3570a1bc | 302 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
303 | return arch_atomic_fetch_dec_acquire(v); |
304 | } | |
aa525d06 | 305 | |
c020395b | 306 | static __always_inline int |
aa525d06 MR |
307 | atomic_fetch_dec_release(atomic_t *v) |
308 | { | |
e87c4f66 | 309 | kcsan_release(); |
3570a1bc | 310 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
311 | return arch_atomic_fetch_dec_release(v); |
312 | } | |
aa525d06 | 313 | |
c020395b | 314 | static __always_inline int |
aa525d06 MR |
315 | atomic_fetch_dec_relaxed(atomic_t *v) |
316 | { | |
3570a1bc | 317 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
318 | return arch_atomic_fetch_dec_relaxed(v); |
319 | } | |
aa525d06 | 320 | |
c020395b | 321 | static __always_inline void |
aa525d06 MR |
322 | atomic_and(int i, atomic_t *v) |
323 | { | |
3570a1bc | 324 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
325 | arch_atomic_and(i, v); |
326 | } | |
aa525d06 | 327 | |
c020395b | 328 | static __always_inline int |
aa525d06 MR |
329 | atomic_fetch_and(int i, atomic_t *v) |
330 | { | |
e87c4f66 | 331 | kcsan_mb(); |
3570a1bc | 332 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
333 | return arch_atomic_fetch_and(i, v); |
334 | } | |
aa525d06 | 335 | |
c020395b | 336 | static __always_inline int |
aa525d06 MR |
337 | atomic_fetch_and_acquire(int i, atomic_t *v) |
338 | { | |
3570a1bc | 339 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
340 | return arch_atomic_fetch_and_acquire(i, v); |
341 | } | |
aa525d06 | 342 | |
c020395b | 343 | static __always_inline int |
aa525d06 MR |
344 | atomic_fetch_and_release(int i, atomic_t *v) |
345 | { | |
e87c4f66 | 346 | kcsan_release(); |
3570a1bc | 347 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
348 | return arch_atomic_fetch_and_release(i, v); |
349 | } | |
aa525d06 | 350 | |
c020395b | 351 | static __always_inline int |
aa525d06 MR |
352 | atomic_fetch_and_relaxed(int i, atomic_t *v) |
353 | { | |
3570a1bc | 354 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
355 | return arch_atomic_fetch_and_relaxed(i, v); |
356 | } | |
aa525d06 | 357 | |
c020395b | 358 | static __always_inline void |
aa525d06 MR |
359 | atomic_andnot(int i, atomic_t *v) |
360 | { | |
3570a1bc | 361 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
362 | arch_atomic_andnot(i, v); |
363 | } | |
aa525d06 | 364 | |
c020395b | 365 | static __always_inline int |
aa525d06 MR |
366 | atomic_fetch_andnot(int i, atomic_t *v) |
367 | { | |
e87c4f66 | 368 | kcsan_mb(); |
3570a1bc | 369 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
370 | return arch_atomic_fetch_andnot(i, v); |
371 | } | |
aa525d06 | 372 | |
c020395b | 373 | static __always_inline int |
aa525d06 MR |
374 | atomic_fetch_andnot_acquire(int i, atomic_t *v) |
375 | { | |
3570a1bc | 376 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
377 | return arch_atomic_fetch_andnot_acquire(i, v); |
378 | } | |
aa525d06 | 379 | |
c020395b | 380 | static __always_inline int |
aa525d06 MR |
381 | atomic_fetch_andnot_release(int i, atomic_t *v) |
382 | { | |
e87c4f66 | 383 | kcsan_release(); |
3570a1bc | 384 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
385 | return arch_atomic_fetch_andnot_release(i, v); |
386 | } | |
aa525d06 | 387 | |
c020395b | 388 | static __always_inline int |
aa525d06 MR |
389 | atomic_fetch_andnot_relaxed(int i, atomic_t *v) |
390 | { | |
3570a1bc | 391 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
392 | return arch_atomic_fetch_andnot_relaxed(i, v); |
393 | } | |
aa525d06 | 394 | |
c020395b | 395 | static __always_inline void |
aa525d06 MR |
396 | atomic_or(int i, atomic_t *v) |
397 | { | |
3570a1bc | 398 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
399 | arch_atomic_or(i, v); |
400 | } | |
aa525d06 | 401 | |
c020395b | 402 | static __always_inline int |
aa525d06 MR |
403 | atomic_fetch_or(int i, atomic_t *v) |
404 | { | |
e87c4f66 | 405 | kcsan_mb(); |
3570a1bc | 406 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
407 | return arch_atomic_fetch_or(i, v); |
408 | } | |
aa525d06 | 409 | |
c020395b | 410 | static __always_inline int |
aa525d06 MR |
411 | atomic_fetch_or_acquire(int i, atomic_t *v) |
412 | { | |
3570a1bc | 413 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
414 | return arch_atomic_fetch_or_acquire(i, v); |
415 | } | |
aa525d06 | 416 | |
c020395b | 417 | static __always_inline int |
aa525d06 MR |
418 | atomic_fetch_or_release(int i, atomic_t *v) |
419 | { | |
e87c4f66 | 420 | kcsan_release(); |
3570a1bc | 421 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
422 | return arch_atomic_fetch_or_release(i, v); |
423 | } | |
aa525d06 | 424 | |
c020395b | 425 | static __always_inline int |
aa525d06 MR |
426 | atomic_fetch_or_relaxed(int i, atomic_t *v) |
427 | { | |
3570a1bc | 428 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
429 | return arch_atomic_fetch_or_relaxed(i, v); |
430 | } | |
aa525d06 | 431 | |
c020395b | 432 | static __always_inline void |
aa525d06 MR |
433 | atomic_xor(int i, atomic_t *v) |
434 | { | |
3570a1bc | 435 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
436 | arch_atomic_xor(i, v); |
437 | } | |
aa525d06 | 438 | |
c020395b | 439 | static __always_inline int |
aa525d06 MR |
440 | atomic_fetch_xor(int i, atomic_t *v) |
441 | { | |
e87c4f66 | 442 | kcsan_mb(); |
3570a1bc | 443 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
444 | return arch_atomic_fetch_xor(i, v); |
445 | } | |
aa525d06 | 446 | |
c020395b | 447 | static __always_inline int |
aa525d06 MR |
448 | atomic_fetch_xor_acquire(int i, atomic_t *v) |
449 | { | |
3570a1bc | 450 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
451 | return arch_atomic_fetch_xor_acquire(i, v); |
452 | } | |
aa525d06 | 453 | |
c020395b | 454 | static __always_inline int |
aa525d06 MR |
455 | atomic_fetch_xor_release(int i, atomic_t *v) |
456 | { | |
e87c4f66 | 457 | kcsan_release(); |
3570a1bc | 458 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
459 | return arch_atomic_fetch_xor_release(i, v); |
460 | } | |
aa525d06 | 461 | |
c020395b | 462 | static __always_inline int |
aa525d06 MR |
463 | atomic_fetch_xor_relaxed(int i, atomic_t *v) |
464 | { | |
3570a1bc | 465 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
466 | return arch_atomic_fetch_xor_relaxed(i, v); |
467 | } | |
aa525d06 | 468 | |
c020395b | 469 | static __always_inline int |
aa525d06 MR |
470 | atomic_xchg(atomic_t *v, int i) |
471 | { | |
e87c4f66 | 472 | kcsan_mb(); |
3570a1bc | 473 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
474 | return arch_atomic_xchg(v, i); |
475 | } | |
aa525d06 | 476 | |
c020395b | 477 | static __always_inline int |
aa525d06 MR |
478 | atomic_xchg_acquire(atomic_t *v, int i) |
479 | { | |
3570a1bc | 480 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
481 | return arch_atomic_xchg_acquire(v, i); |
482 | } | |
aa525d06 | 483 | |
c020395b | 484 | static __always_inline int |
aa525d06 MR |
485 | atomic_xchg_release(atomic_t *v, int i) |
486 | { | |
e87c4f66 | 487 | kcsan_release(); |
3570a1bc | 488 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
489 | return arch_atomic_xchg_release(v, i); |
490 | } | |
aa525d06 | 491 | |
c020395b | 492 | static __always_inline int |
aa525d06 MR |
493 | atomic_xchg_relaxed(atomic_t *v, int i) |
494 | { | |
3570a1bc | 495 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
496 | return arch_atomic_xchg_relaxed(v, i); |
497 | } | |
aa525d06 | 498 | |
c020395b | 499 | static __always_inline int |
aa525d06 MR |
500 | atomic_cmpxchg(atomic_t *v, int old, int new) |
501 | { | |
e87c4f66 | 502 | kcsan_mb(); |
3570a1bc | 503 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
504 | return arch_atomic_cmpxchg(v, old, new); |
505 | } | |
aa525d06 | 506 | |
c020395b | 507 | static __always_inline int |
aa525d06 MR |
508 | atomic_cmpxchg_acquire(atomic_t *v, int old, int new) |
509 | { | |
3570a1bc | 510 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
511 | return arch_atomic_cmpxchg_acquire(v, old, new); |
512 | } | |
aa525d06 | 513 | |
c020395b | 514 | static __always_inline int |
aa525d06 MR |
515 | atomic_cmpxchg_release(atomic_t *v, int old, int new) |
516 | { | |
e87c4f66 | 517 | kcsan_release(); |
3570a1bc | 518 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
519 | return arch_atomic_cmpxchg_release(v, old, new); |
520 | } | |
aa525d06 | 521 | |
c020395b | 522 | static __always_inline int |
aa525d06 MR |
523 | atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) |
524 | { | |
3570a1bc | 525 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
526 | return arch_atomic_cmpxchg_relaxed(v, old, new); |
527 | } | |
aa525d06 | 528 | |
c020395b | 529 | static __always_inline bool |
aa525d06 MR |
530 | atomic_try_cmpxchg(atomic_t *v, int *old, int new) |
531 | { | |
e87c4f66 | 532 | kcsan_mb(); |
3570a1bc ME |
533 | instrument_atomic_read_write(v, sizeof(*v)); |
534 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 MR |
535 | return arch_atomic_try_cmpxchg(v, old, new); |
536 | } | |
aa525d06 | 537 | |
c020395b | 538 | static __always_inline bool |
aa525d06 MR |
539 | atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) |
540 | { | |
3570a1bc ME |
541 | instrument_atomic_read_write(v, sizeof(*v)); |
542 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 MR |
543 | return arch_atomic_try_cmpxchg_acquire(v, old, new); |
544 | } | |
aa525d06 | 545 | |
c020395b | 546 | static __always_inline bool |
aa525d06 MR |
547 | atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) |
548 | { | |
e87c4f66 | 549 | kcsan_release(); |
3570a1bc ME |
550 | instrument_atomic_read_write(v, sizeof(*v)); |
551 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 MR |
552 | return arch_atomic_try_cmpxchg_release(v, old, new); |
553 | } | |
aa525d06 | 554 | |
c020395b | 555 | static __always_inline bool |
aa525d06 MR |
556 | atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) |
557 | { | |
3570a1bc ME |
558 | instrument_atomic_read_write(v, sizeof(*v)); |
559 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 MR |
560 | return arch_atomic_try_cmpxchg_relaxed(v, old, new); |
561 | } | |
aa525d06 | 562 | |
c020395b | 563 | static __always_inline bool |
aa525d06 MR |
564 | atomic_sub_and_test(int i, atomic_t *v) |
565 | { | |
e87c4f66 | 566 | kcsan_mb(); |
3570a1bc | 567 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
568 | return arch_atomic_sub_and_test(i, v); |
569 | } | |
aa525d06 | 570 | |
c020395b | 571 | static __always_inline bool |
aa525d06 MR |
572 | atomic_dec_and_test(atomic_t *v) |
573 | { | |
e87c4f66 | 574 | kcsan_mb(); |
3570a1bc | 575 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
576 | return arch_atomic_dec_and_test(v); |
577 | } | |
aa525d06 | 578 | |
c020395b | 579 | static __always_inline bool |
aa525d06 MR |
580 | atomic_inc_and_test(atomic_t *v) |
581 | { | |
e87c4f66 | 582 | kcsan_mb(); |
3570a1bc | 583 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
584 | return arch_atomic_inc_and_test(v); |
585 | } | |
aa525d06 | 586 | |
c020395b | 587 | static __always_inline bool |
aa525d06 MR |
588 | atomic_add_negative(int i, atomic_t *v) |
589 | { | |
e87c4f66 | 590 | kcsan_mb(); |
3570a1bc | 591 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
592 | return arch_atomic_add_negative(i, v); |
593 | } | |
aa525d06 | 594 | |
e5ab9eff TG |
595 | static __always_inline bool |
596 | atomic_add_negative_acquire(int i, atomic_t *v) | |
597 | { | |
598 | instrument_atomic_read_write(v, sizeof(*v)); | |
599 | return arch_atomic_add_negative_acquire(i, v); | |
600 | } | |
601 | ||
602 | static __always_inline bool | |
603 | atomic_add_negative_release(int i, atomic_t *v) | |
604 | { | |
605 | kcsan_release(); | |
606 | instrument_atomic_read_write(v, sizeof(*v)); | |
607 | return arch_atomic_add_negative_release(i, v); | |
608 | } | |
609 | ||
610 | static __always_inline bool | |
611 | atomic_add_negative_relaxed(int i, atomic_t *v) | |
612 | { | |
613 | instrument_atomic_read_write(v, sizeof(*v)); | |
614 | return arch_atomic_add_negative_relaxed(i, v); | |
615 | } | |
616 | ||
c020395b | 617 | static __always_inline int |
aa525d06 MR |
618 | atomic_fetch_add_unless(atomic_t *v, int a, int u) |
619 | { | |
e87c4f66 | 620 | kcsan_mb(); |
3570a1bc | 621 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
622 | return arch_atomic_fetch_add_unless(v, a, u); |
623 | } | |
aa525d06 | 624 | |
c020395b | 625 | static __always_inline bool |
aa525d06 MR |
626 | atomic_add_unless(atomic_t *v, int a, int u) |
627 | { | |
e87c4f66 | 628 | kcsan_mb(); |
3570a1bc | 629 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
630 | return arch_atomic_add_unless(v, a, u); |
631 | } | |
aa525d06 | 632 | |
c020395b | 633 | static __always_inline bool |
aa525d06 MR |
634 | atomic_inc_not_zero(atomic_t *v) |
635 | { | |
e87c4f66 | 636 | kcsan_mb(); |
3570a1bc | 637 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
638 | return arch_atomic_inc_not_zero(v); |
639 | } | |
aa525d06 | 640 | |
c020395b | 641 | static __always_inline bool |
aa525d06 MR |
642 | atomic_inc_unless_negative(atomic_t *v) |
643 | { | |
e87c4f66 | 644 | kcsan_mb(); |
3570a1bc | 645 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
646 | return arch_atomic_inc_unless_negative(v); |
647 | } | |
aa525d06 | 648 | |
c020395b | 649 | static __always_inline bool |
aa525d06 MR |
650 | atomic_dec_unless_positive(atomic_t *v) |
651 | { | |
e87c4f66 | 652 | kcsan_mb(); |
3570a1bc | 653 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
654 | return arch_atomic_dec_unless_positive(v); |
655 | } | |
aa525d06 | 656 | |
c020395b | 657 | static __always_inline int |
aa525d06 MR |
658 | atomic_dec_if_positive(atomic_t *v) |
659 | { | |
e87c4f66 | 660 | kcsan_mb(); |
3570a1bc | 661 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
662 | return arch_atomic_dec_if_positive(v); |
663 | } | |
aa525d06 | 664 | |
c020395b | 665 | static __always_inline s64 |
aa525d06 MR |
666 | atomic64_read(const atomic64_t *v) |
667 | { | |
ed8af2e4 | 668 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
669 | return arch_atomic64_read(v); |
670 | } | |
aa525d06 | 671 | |
c020395b | 672 | static __always_inline s64 |
aa525d06 MR |
673 | atomic64_read_acquire(const atomic64_t *v) |
674 | { | |
ed8af2e4 | 675 | instrument_atomic_read(v, sizeof(*v)); |
aa525d06 MR |
676 | return arch_atomic64_read_acquire(v); |
677 | } | |
aa525d06 | 678 | |
c020395b | 679 | static __always_inline void |
aa525d06 MR |
680 | atomic64_set(atomic64_t *v, s64 i) |
681 | { | |
ed8af2e4 | 682 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
683 | arch_atomic64_set(v, i); |
684 | } | |
aa525d06 | 685 | |
c020395b | 686 | static __always_inline void |
aa525d06 MR |
687 | atomic64_set_release(atomic64_t *v, s64 i) |
688 | { | |
e87c4f66 | 689 | kcsan_release(); |
ed8af2e4 | 690 | instrument_atomic_write(v, sizeof(*v)); |
aa525d06 MR |
691 | arch_atomic64_set_release(v, i); |
692 | } | |
aa525d06 | 693 | |
c020395b | 694 | static __always_inline void |
aa525d06 MR |
695 | atomic64_add(s64 i, atomic64_t *v) |
696 | { | |
3570a1bc | 697 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
698 | arch_atomic64_add(i, v); |
699 | } | |
aa525d06 | 700 | |
c020395b | 701 | static __always_inline s64 |
aa525d06 MR |
702 | atomic64_add_return(s64 i, atomic64_t *v) |
703 | { | |
e87c4f66 | 704 | kcsan_mb(); |
3570a1bc | 705 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
706 | return arch_atomic64_add_return(i, v); |
707 | } | |
aa525d06 | 708 | |
c020395b | 709 | static __always_inline s64 |
aa525d06 MR |
710 | atomic64_add_return_acquire(s64 i, atomic64_t *v) |
711 | { | |
3570a1bc | 712 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
713 | return arch_atomic64_add_return_acquire(i, v); |
714 | } | |
aa525d06 | 715 | |
c020395b | 716 | static __always_inline s64 |
aa525d06 MR |
717 | atomic64_add_return_release(s64 i, atomic64_t *v) |
718 | { | |
e87c4f66 | 719 | kcsan_release(); |
3570a1bc | 720 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
721 | return arch_atomic64_add_return_release(i, v); |
722 | } | |
aa525d06 | 723 | |
c020395b | 724 | static __always_inline s64 |
aa525d06 MR |
725 | atomic64_add_return_relaxed(s64 i, atomic64_t *v) |
726 | { | |
3570a1bc | 727 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
728 | return arch_atomic64_add_return_relaxed(i, v); |
729 | } | |
aa525d06 | 730 | |
c020395b | 731 | static __always_inline s64 |
aa525d06 MR |
732 | atomic64_fetch_add(s64 i, atomic64_t *v) |
733 | { | |
e87c4f66 | 734 | kcsan_mb(); |
3570a1bc | 735 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
736 | return arch_atomic64_fetch_add(i, v); |
737 | } | |
aa525d06 | 738 | |
c020395b | 739 | static __always_inline s64 |
aa525d06 MR |
740 | atomic64_fetch_add_acquire(s64 i, atomic64_t *v) |
741 | { | |
3570a1bc | 742 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
743 | return arch_atomic64_fetch_add_acquire(i, v); |
744 | } | |
aa525d06 | 745 | |
c020395b | 746 | static __always_inline s64 |
aa525d06 MR |
747 | atomic64_fetch_add_release(s64 i, atomic64_t *v) |
748 | { | |
e87c4f66 | 749 | kcsan_release(); |
3570a1bc | 750 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
751 | return arch_atomic64_fetch_add_release(i, v); |
752 | } | |
aa525d06 | 753 | |
c020395b | 754 | static __always_inline s64 |
aa525d06 MR |
755 | atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) |
756 | { | |
3570a1bc | 757 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
758 | return arch_atomic64_fetch_add_relaxed(i, v); |
759 | } | |
aa525d06 | 760 | |
c020395b | 761 | static __always_inline void |
aa525d06 MR |
762 | atomic64_sub(s64 i, atomic64_t *v) |
763 | { | |
3570a1bc | 764 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
765 | arch_atomic64_sub(i, v); |
766 | } | |
ac605bee | 767 | |
c020395b | 768 | static __always_inline s64 |
aa525d06 MR |
769 | atomic64_sub_return(s64 i, atomic64_t *v) |
770 | { | |
e87c4f66 | 771 | kcsan_mb(); |
3570a1bc | 772 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
773 | return arch_atomic64_sub_return(i, v); |
774 | } | |
b06ed71a | 775 | |
c020395b | 776 | static __always_inline s64 |
aa525d06 MR |
777 | atomic64_sub_return_acquire(s64 i, atomic64_t *v) |
778 | { | |
3570a1bc | 779 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
780 | return arch_atomic64_sub_return_acquire(i, v); |
781 | } | |
b06ed71a | 782 | |
c020395b | 783 | static __always_inline s64 |
aa525d06 | 784 | atomic64_sub_return_release(s64 i, atomic64_t *v) |
b06ed71a | 785 | { |
e87c4f66 | 786 | kcsan_release(); |
3570a1bc | 787 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 788 | return arch_atomic64_sub_return_release(i, v); |
b06ed71a DV |
789 | } |
790 | ||
c020395b | 791 | static __always_inline s64 |
aa525d06 | 792 | atomic64_sub_return_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 793 | { |
3570a1bc | 794 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 795 | return arch_atomic64_sub_return_relaxed(i, v); |
b06ed71a DV |
796 | } |
797 | ||
c020395b | 798 | static __always_inline s64 |
aa525d06 | 799 | atomic64_fetch_sub(s64 i, atomic64_t *v) |
b06ed71a | 800 | { |
e87c4f66 | 801 | kcsan_mb(); |
3570a1bc | 802 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 803 | return arch_atomic64_fetch_sub(i, v); |
b06ed71a DV |
804 | } |
805 | ||
c020395b | 806 | static __always_inline s64 |
aa525d06 | 807 | atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) |
b06ed71a | 808 | { |
3570a1bc | 809 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 810 | return arch_atomic64_fetch_sub_acquire(i, v); |
b06ed71a DV |
811 | } |
812 | ||
c020395b | 813 | static __always_inline s64 |
aa525d06 | 814 | atomic64_fetch_sub_release(s64 i, atomic64_t *v) |
b06ed71a | 815 | { |
e87c4f66 | 816 | kcsan_release(); |
3570a1bc | 817 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 818 | return arch_atomic64_fetch_sub_release(i, v); |
b06ed71a DV |
819 | } |
820 | ||
c020395b | 821 | static __always_inline s64 |
aa525d06 | 822 | atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 823 | { |
3570a1bc | 824 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 825 | return arch_atomic64_fetch_sub_relaxed(i, v); |
b06ed71a DV |
826 | } |
827 | ||
c020395b | 828 | static __always_inline void |
aa525d06 | 829 | atomic64_inc(atomic64_t *v) |
b06ed71a | 830 | { |
3570a1bc | 831 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 832 | arch_atomic64_inc(v); |
b06ed71a DV |
833 | } |
834 | ||
c020395b | 835 | static __always_inline s64 |
aa525d06 | 836 | atomic64_inc_return(atomic64_t *v) |
b06ed71a | 837 | { |
e87c4f66 | 838 | kcsan_mb(); |
3570a1bc | 839 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 840 | return arch_atomic64_inc_return(v); |
b06ed71a DV |
841 | } |
842 | ||
c020395b | 843 | static __always_inline s64 |
aa525d06 | 844 | atomic64_inc_return_acquire(atomic64_t *v) |
b06ed71a | 845 | { |
3570a1bc | 846 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 847 | return arch_atomic64_inc_return_acquire(v); |
b06ed71a | 848 | } |
b06ed71a | 849 | |
c020395b | 850 | static __always_inline s64 |
aa525d06 | 851 | atomic64_inc_return_release(atomic64_t *v) |
b06ed71a | 852 | { |
e87c4f66 | 853 | kcsan_release(); |
3570a1bc | 854 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 855 | return arch_atomic64_inc_return_release(v); |
b06ed71a | 856 | } |
b06ed71a | 857 | |
c020395b | 858 | static __always_inline s64 |
aa525d06 | 859 | atomic64_inc_return_relaxed(atomic64_t *v) |
b06ed71a | 860 | { |
3570a1bc | 861 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 862 | return arch_atomic64_inc_return_relaxed(v); |
b06ed71a | 863 | } |
b06ed71a | 864 | |
c020395b | 865 | static __always_inline s64 |
aa525d06 | 866 | atomic64_fetch_inc(atomic64_t *v) |
0ae1d994 | 867 | { |
e87c4f66 | 868 | kcsan_mb(); |
3570a1bc | 869 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 870 | return arch_atomic64_fetch_inc(v); |
0ae1d994 | 871 | } |
b06ed71a | 872 | |
c020395b | 873 | static __always_inline s64 |
aa525d06 | 874 | atomic64_fetch_inc_acquire(atomic64_t *v) |
b06ed71a | 875 | { |
3570a1bc | 876 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 877 | return arch_atomic64_fetch_inc_acquire(v); |
b06ed71a DV |
878 | } |
879 | ||
c020395b | 880 | static __always_inline s64 |
aa525d06 | 881 | atomic64_fetch_inc_release(atomic64_t *v) |
b06ed71a | 882 | { |
e87c4f66 | 883 | kcsan_release(); |
3570a1bc | 884 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 885 | return arch_atomic64_fetch_inc_release(v); |
b06ed71a DV |
886 | } |
887 | ||
c020395b | 888 | static __always_inline s64 |
aa525d06 | 889 | atomic64_fetch_inc_relaxed(atomic64_t *v) |
b06ed71a | 890 | { |
3570a1bc | 891 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 892 | return arch_atomic64_fetch_inc_relaxed(v); |
b06ed71a DV |
893 | } |
894 | ||
c020395b | 895 | static __always_inline void |
aa525d06 | 896 | atomic64_dec(atomic64_t *v) |
b06ed71a | 897 | { |
3570a1bc | 898 | instrument_atomic_read_write(v, sizeof(*v)); |
b06ed71a DV |
899 | arch_atomic64_dec(v); |
900 | } | |
901 | ||
c020395b | 902 | static __always_inline s64 |
aa525d06 | 903 | atomic64_dec_return(atomic64_t *v) |
b06ed71a | 904 | { |
e87c4f66 | 905 | kcsan_mb(); |
3570a1bc | 906 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 907 | return arch_atomic64_dec_return(v); |
b06ed71a DV |
908 | } |
909 | ||
c020395b | 910 | static __always_inline s64 |
aa525d06 | 911 | atomic64_dec_return_acquire(atomic64_t *v) |
b06ed71a | 912 | { |
3570a1bc | 913 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 914 | return arch_atomic64_dec_return_acquire(v); |
b06ed71a DV |
915 | } |
916 | ||
c020395b | 917 | static __always_inline s64 |
aa525d06 | 918 | atomic64_dec_return_release(atomic64_t *v) |
b06ed71a | 919 | { |
e87c4f66 | 920 | kcsan_release(); |
3570a1bc | 921 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 922 | return arch_atomic64_dec_return_release(v); |
b06ed71a DV |
923 | } |
924 | ||
c020395b | 925 | static __always_inline s64 |
aa525d06 | 926 | atomic64_dec_return_relaxed(atomic64_t *v) |
b06ed71a | 927 | { |
3570a1bc | 928 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 929 | return arch_atomic64_dec_return_relaxed(v); |
b06ed71a DV |
930 | } |
931 | ||
c020395b | 932 | static __always_inline s64 |
aa525d06 | 933 | atomic64_fetch_dec(atomic64_t *v) |
b06ed71a | 934 | { |
e87c4f66 | 935 | kcsan_mb(); |
3570a1bc | 936 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
937 | return arch_atomic64_fetch_dec(v); |
938 | } | |
aa525d06 | 939 | |
c020395b | 940 | static __always_inline s64 |
aa525d06 MR |
941 | atomic64_fetch_dec_acquire(atomic64_t *v) |
942 | { | |
3570a1bc | 943 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
944 | return arch_atomic64_fetch_dec_acquire(v); |
945 | } | |
aa525d06 | 946 | |
c020395b | 947 | static __always_inline s64 |
aa525d06 MR |
948 | atomic64_fetch_dec_release(atomic64_t *v) |
949 | { | |
e87c4f66 | 950 | kcsan_release(); |
3570a1bc | 951 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 952 | return arch_atomic64_fetch_dec_release(v); |
b06ed71a | 953 | } |
aa525d06 | 954 | |
c020395b | 955 | static __always_inline s64 |
aa525d06 MR |
956 | atomic64_fetch_dec_relaxed(atomic64_t *v) |
957 | { | |
3570a1bc | 958 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
959 | return arch_atomic64_fetch_dec_relaxed(v); |
960 | } | |
b06ed71a | 961 | |
c020395b | 962 | static __always_inline void |
aa525d06 | 963 | atomic64_and(s64 i, atomic64_t *v) |
b06ed71a | 964 | { |
3570a1bc | 965 | instrument_atomic_read_write(v, sizeof(*v)); |
b06ed71a DV |
966 | arch_atomic64_and(i, v); |
967 | } | |
968 | ||
c020395b | 969 | static __always_inline s64 |
aa525d06 | 970 | atomic64_fetch_and(s64 i, atomic64_t *v) |
b06ed71a | 971 | { |
e87c4f66 | 972 | kcsan_mb(); |
3570a1bc | 973 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 974 | return arch_atomic64_fetch_and(i, v); |
b06ed71a DV |
975 | } |
976 | ||
c020395b | 977 | static __always_inline s64 |
aa525d06 | 978 | atomic64_fetch_and_acquire(s64 i, atomic64_t *v) |
b06ed71a | 979 | { |
3570a1bc | 980 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 981 | return arch_atomic64_fetch_and_acquire(i, v); |
b06ed71a DV |
982 | } |
983 | ||
c020395b | 984 | static __always_inline s64 |
aa525d06 | 985 | atomic64_fetch_and_release(s64 i, atomic64_t *v) |
b06ed71a | 986 | { |
e87c4f66 | 987 | kcsan_release(); |
3570a1bc | 988 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 989 | return arch_atomic64_fetch_and_release(i, v); |
b06ed71a DV |
990 | } |
991 | ||
c020395b | 992 | static __always_inline s64 |
aa525d06 | 993 | atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 994 | { |
3570a1bc | 995 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 996 | return arch_atomic64_fetch_and_relaxed(i, v); |
b06ed71a DV |
997 | } |
998 | ||
c020395b | 999 | static __always_inline void |
aa525d06 | 1000 | atomic64_andnot(s64 i, atomic64_t *v) |
b06ed71a | 1001 | { |
3570a1bc | 1002 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1003 | arch_atomic64_andnot(i, v); |
b06ed71a DV |
1004 | } |
1005 | ||
c020395b | 1006 | static __always_inline s64 |
aa525d06 | 1007 | atomic64_fetch_andnot(s64 i, atomic64_t *v) |
b06ed71a | 1008 | { |
e87c4f66 | 1009 | kcsan_mb(); |
3570a1bc | 1010 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1011 | return arch_atomic64_fetch_andnot(i, v); |
b06ed71a DV |
1012 | } |
1013 | ||
c020395b | 1014 | static __always_inline s64 |
aa525d06 | 1015 | atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) |
b06ed71a | 1016 | { |
3570a1bc | 1017 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1018 | return arch_atomic64_fetch_andnot_acquire(i, v); |
b06ed71a DV |
1019 | } |
1020 | ||
c020395b | 1021 | static __always_inline s64 |
aa525d06 | 1022 | atomic64_fetch_andnot_release(s64 i, atomic64_t *v) |
b06ed71a | 1023 | { |
e87c4f66 | 1024 | kcsan_release(); |
3570a1bc | 1025 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1026 | return arch_atomic64_fetch_andnot_release(i, v); |
b06ed71a DV |
1027 | } |
1028 | ||
c020395b | 1029 | static __always_inline s64 |
aa525d06 | 1030 | atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 1031 | { |
3570a1bc | 1032 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1033 | return arch_atomic64_fetch_andnot_relaxed(i, v); |
b06ed71a DV |
1034 | } |
1035 | ||
c020395b | 1036 | static __always_inline void |
aa525d06 | 1037 | atomic64_or(s64 i, atomic64_t *v) |
b06ed71a | 1038 | { |
3570a1bc | 1039 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1040 | arch_atomic64_or(i, v); |
1041 | } | |
aa525d06 | 1042 | |
c020395b | 1043 | static __always_inline s64 |
aa525d06 MR |
1044 | atomic64_fetch_or(s64 i, atomic64_t *v) |
1045 | { | |
e87c4f66 | 1046 | kcsan_mb(); |
3570a1bc | 1047 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1048 | return arch_atomic64_fetch_or(i, v); |
b06ed71a DV |
1049 | } |
1050 | ||
c020395b | 1051 | static __always_inline s64 |
aa525d06 | 1052 | atomic64_fetch_or_acquire(s64 i, atomic64_t *v) |
b06ed71a | 1053 | { |
3570a1bc | 1054 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1055 | return arch_atomic64_fetch_or_acquire(i, v); |
b06ed71a DV |
1056 | } |
1057 | ||
c020395b | 1058 | static __always_inline s64 |
aa525d06 | 1059 | atomic64_fetch_or_release(s64 i, atomic64_t *v) |
b06ed71a | 1060 | { |
e87c4f66 | 1061 | kcsan_release(); |
3570a1bc | 1062 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1063 | return arch_atomic64_fetch_or_release(i, v); |
b06ed71a DV |
1064 | } |
1065 | ||
c020395b | 1066 | static __always_inline s64 |
aa525d06 | 1067 | atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 1068 | { |
3570a1bc | 1069 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1070 | return arch_atomic64_fetch_or_relaxed(i, v); |
b06ed71a DV |
1071 | } |
1072 | ||
c020395b | 1073 | static __always_inline void |
aa525d06 | 1074 | atomic64_xor(s64 i, atomic64_t *v) |
b06ed71a | 1075 | { |
3570a1bc | 1076 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1077 | arch_atomic64_xor(i, v); |
1078 | } | |
aa525d06 | 1079 | |
c020395b | 1080 | static __always_inline s64 |
aa525d06 MR |
1081 | atomic64_fetch_xor(s64 i, atomic64_t *v) |
1082 | { | |
e87c4f66 | 1083 | kcsan_mb(); |
3570a1bc | 1084 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1085 | return arch_atomic64_fetch_xor(i, v); |
b06ed71a DV |
1086 | } |
1087 | ||
c020395b | 1088 | static __always_inline s64 |
aa525d06 | 1089 | atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) |
b06ed71a | 1090 | { |
3570a1bc | 1091 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1092 | return arch_atomic64_fetch_xor_acquire(i, v); |
b06ed71a DV |
1093 | } |
1094 | ||
c020395b | 1095 | static __always_inline s64 |
aa525d06 | 1096 | atomic64_fetch_xor_release(s64 i, atomic64_t *v) |
b06ed71a | 1097 | { |
e87c4f66 | 1098 | kcsan_release(); |
3570a1bc | 1099 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1100 | return arch_atomic64_fetch_xor_release(i, v); |
b06ed71a DV |
1101 | } |
1102 | ||
c020395b | 1103 | static __always_inline s64 |
aa525d06 | 1104 | atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) |
b06ed71a | 1105 | { |
3570a1bc | 1106 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1107 | return arch_atomic64_fetch_xor_relaxed(i, v); |
b06ed71a DV |
1108 | } |
1109 | ||
c020395b | 1110 | static __always_inline s64 |
aa525d06 | 1111 | atomic64_xchg(atomic64_t *v, s64 i) |
b06ed71a | 1112 | { |
e87c4f66 | 1113 | kcsan_mb(); |
3570a1bc | 1114 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1115 | return arch_atomic64_xchg(v, i); |
b06ed71a DV |
1116 | } |
1117 | ||
c020395b | 1118 | static __always_inline s64 |
aa525d06 | 1119 | atomic64_xchg_acquire(atomic64_t *v, s64 i) |
b06ed71a | 1120 | { |
3570a1bc | 1121 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1122 | return arch_atomic64_xchg_acquire(v, i); |
b06ed71a DV |
1123 | } |
1124 | ||
c020395b | 1125 | static __always_inline s64 |
aa525d06 | 1126 | atomic64_xchg_release(atomic64_t *v, s64 i) |
b06ed71a | 1127 | { |
e87c4f66 | 1128 | kcsan_release(); |
3570a1bc | 1129 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1130 | return arch_atomic64_xchg_release(v, i); |
b06ed71a DV |
1131 | } |
1132 | ||
c020395b | 1133 | static __always_inline s64 |
aa525d06 | 1134 | atomic64_xchg_relaxed(atomic64_t *v, s64 i) |
b06ed71a | 1135 | { |
3570a1bc | 1136 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1137 | return arch_atomic64_xchg_relaxed(v, i); |
b06ed71a DV |
1138 | } |
1139 | ||
c020395b | 1140 | static __always_inline s64 |
aa525d06 | 1141 | atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1142 | { |
e87c4f66 | 1143 | kcsan_mb(); |
3570a1bc | 1144 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1145 | return arch_atomic64_cmpxchg(v, old, new); |
b06ed71a DV |
1146 | } |
1147 | ||
c020395b | 1148 | static __always_inline s64 |
aa525d06 | 1149 | atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1150 | { |
3570a1bc | 1151 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1152 | return arch_atomic64_cmpxchg_acquire(v, old, new); |
b06ed71a DV |
1153 | } |
1154 | ||
c020395b | 1155 | static __always_inline s64 |
aa525d06 | 1156 | atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1157 | { |
e87c4f66 | 1158 | kcsan_release(); |
3570a1bc | 1159 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1160 | return arch_atomic64_cmpxchg_release(v, old, new); |
b06ed71a DV |
1161 | } |
1162 | ||
c020395b | 1163 | static __always_inline s64 |
aa525d06 | 1164 | atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) |
b06ed71a | 1165 | { |
3570a1bc | 1166 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1167 | return arch_atomic64_cmpxchg_relaxed(v, old, new); |
b06ed71a DV |
1168 | } |
1169 | ||
c020395b | 1170 | static __always_inline bool |
aa525d06 | 1171 | atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1172 | { |
e87c4f66 | 1173 | kcsan_mb(); |
3570a1bc ME |
1174 | instrument_atomic_read_write(v, sizeof(*v)); |
1175 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 | 1176 | return arch_atomic64_try_cmpxchg(v, old, new); |
b06ed71a DV |
1177 | } |
1178 | ||
c020395b | 1179 | static __always_inline bool |
aa525d06 | 1180 | atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1181 | { |
3570a1bc ME |
1182 | instrument_atomic_read_write(v, sizeof(*v)); |
1183 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 | 1184 | return arch_atomic64_try_cmpxchg_acquire(v, old, new); |
b06ed71a DV |
1185 | } |
1186 | ||
c020395b | 1187 | static __always_inline bool |
aa525d06 | 1188 | atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1189 | { |
e87c4f66 | 1190 | kcsan_release(); |
3570a1bc ME |
1191 | instrument_atomic_read_write(v, sizeof(*v)); |
1192 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 | 1193 | return arch_atomic64_try_cmpxchg_release(v, old, new); |
b06ed71a DV |
1194 | } |
1195 | ||
c020395b | 1196 | static __always_inline bool |
aa525d06 | 1197 | atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) |
b06ed71a | 1198 | { |
3570a1bc ME |
1199 | instrument_atomic_read_write(v, sizeof(*v)); |
1200 | instrument_atomic_read_write(old, sizeof(*old)); | |
aa525d06 | 1201 | return arch_atomic64_try_cmpxchg_relaxed(v, old, new); |
b06ed71a DV |
1202 | } |
1203 | ||
c020395b | 1204 | static __always_inline bool |
aa525d06 | 1205 | atomic64_sub_and_test(s64 i, atomic64_t *v) |
b06ed71a | 1206 | { |
e87c4f66 | 1207 | kcsan_mb(); |
3570a1bc | 1208 | instrument_atomic_read_write(v, sizeof(*v)); |
b06ed71a DV |
1209 | return arch_atomic64_sub_and_test(i, v); |
1210 | } | |
1211 | ||
c020395b | 1212 | static __always_inline bool |
aa525d06 | 1213 | atomic64_dec_and_test(atomic64_t *v) |
b06ed71a | 1214 | { |
e87c4f66 | 1215 | kcsan_mb(); |
3570a1bc | 1216 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 | 1217 | return arch_atomic64_dec_and_test(v); |
b06ed71a DV |
1218 | } |
1219 | ||
c020395b | 1220 | static __always_inline bool |
aa525d06 MR |
1221 | atomic64_inc_and_test(atomic64_t *v) |
1222 | { | |
e87c4f66 | 1223 | kcsan_mb(); |
3570a1bc | 1224 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1225 | return arch_atomic64_inc_and_test(v); |
1226 | } | |
aa525d06 | 1227 | |
c020395b | 1228 | static __always_inline bool |
aa525d06 | 1229 | atomic64_add_negative(s64 i, atomic64_t *v) |
b06ed71a | 1230 | { |
e87c4f66 | 1231 | kcsan_mb(); |
3570a1bc | 1232 | instrument_atomic_read_write(v, sizeof(*v)); |
b06ed71a DV |
1233 | return arch_atomic64_add_negative(i, v); |
1234 | } | |
aa525d06 | 1235 | |
e5ab9eff TG |
1236 | static __always_inline bool |
1237 | atomic64_add_negative_acquire(s64 i, atomic64_t *v) | |
1238 | { | |
1239 | instrument_atomic_read_write(v, sizeof(*v)); | |
1240 | return arch_atomic64_add_negative_acquire(i, v); | |
1241 | } | |
1242 | ||
1243 | static __always_inline bool | |
1244 | atomic64_add_negative_release(s64 i, atomic64_t *v) | |
1245 | { | |
1246 | kcsan_release(); | |
1247 | instrument_atomic_read_write(v, sizeof(*v)); | |
1248 | return arch_atomic64_add_negative_release(i, v); | |
1249 | } | |
1250 | ||
1251 | static __always_inline bool | |
1252 | atomic64_add_negative_relaxed(s64 i, atomic64_t *v) | |
1253 | { | |
1254 | instrument_atomic_read_write(v, sizeof(*v)); | |
1255 | return arch_atomic64_add_negative_relaxed(i, v); | |
1256 | } | |
1257 | ||
c020395b | 1258 | static __always_inline s64 |
aa525d06 MR |
1259 | atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) |
1260 | { | |
e87c4f66 | 1261 | kcsan_mb(); |
3570a1bc | 1262 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1263 | return arch_atomic64_fetch_add_unless(v, a, u); |
1264 | } | |
aa525d06 | 1265 | |
c020395b | 1266 | static __always_inline bool |
aa525d06 MR |
1267 | atomic64_add_unless(atomic64_t *v, s64 a, s64 u) |
1268 | { | |
e87c4f66 | 1269 | kcsan_mb(); |
3570a1bc | 1270 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1271 | return arch_atomic64_add_unless(v, a, u); |
1272 | } | |
aa525d06 | 1273 | |
c020395b | 1274 | static __always_inline bool |
aa525d06 MR |
1275 | atomic64_inc_not_zero(atomic64_t *v) |
1276 | { | |
e87c4f66 | 1277 | kcsan_mb(); |
3570a1bc | 1278 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1279 | return arch_atomic64_inc_not_zero(v); |
1280 | } | |
aa525d06 | 1281 | |
c020395b | 1282 | static __always_inline bool |
aa525d06 MR |
1283 | atomic64_inc_unless_negative(atomic64_t *v) |
1284 | { | |
e87c4f66 | 1285 | kcsan_mb(); |
3570a1bc | 1286 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1287 | return arch_atomic64_inc_unless_negative(v); |
1288 | } | |
b06ed71a | 1289 | |
c020395b | 1290 | static __always_inline bool |
aa525d06 MR |
1291 | atomic64_dec_unless_positive(atomic64_t *v) |
1292 | { | |
e87c4f66 | 1293 | kcsan_mb(); |
3570a1bc | 1294 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1295 | return arch_atomic64_dec_unless_positive(v); |
1296 | } | |
aa525d06 | 1297 | |
c020395b | 1298 | static __always_inline s64 |
aa525d06 MR |
1299 | atomic64_dec_if_positive(atomic64_t *v) |
1300 | { | |
e87c4f66 | 1301 | kcsan_mb(); |
3570a1bc | 1302 | instrument_atomic_read_write(v, sizeof(*v)); |
aa525d06 MR |
1303 | return arch_atomic64_dec_if_positive(v); |
1304 | } | |
aa525d06 | 1305 | |
67d1b0de MR |
1306 | static __always_inline long |
1307 | atomic_long_read(const atomic_long_t *v) | |
1308 | { | |
1309 | instrument_atomic_read(v, sizeof(*v)); | |
1310 | return arch_atomic_long_read(v); | |
1311 | } | |
1312 | ||
1313 | static __always_inline long | |
1314 | atomic_long_read_acquire(const atomic_long_t *v) | |
1315 | { | |
1316 | instrument_atomic_read(v, sizeof(*v)); | |
1317 | return arch_atomic_long_read_acquire(v); | |
1318 | } | |
1319 | ||
1320 | static __always_inline void | |
1321 | atomic_long_set(atomic_long_t *v, long i) | |
1322 | { | |
1323 | instrument_atomic_write(v, sizeof(*v)); | |
1324 | arch_atomic_long_set(v, i); | |
1325 | } | |
1326 | ||
1327 | static __always_inline void | |
1328 | atomic_long_set_release(atomic_long_t *v, long i) | |
1329 | { | |
e87c4f66 | 1330 | kcsan_release(); |
67d1b0de MR |
1331 | instrument_atomic_write(v, sizeof(*v)); |
1332 | arch_atomic_long_set_release(v, i); | |
1333 | } | |
1334 | ||
1335 | static __always_inline void | |
1336 | atomic_long_add(long i, atomic_long_t *v) | |
1337 | { | |
1338 | instrument_atomic_read_write(v, sizeof(*v)); | |
1339 | arch_atomic_long_add(i, v); | |
1340 | } | |
1341 | ||
1342 | static __always_inline long | |
1343 | atomic_long_add_return(long i, atomic_long_t *v) | |
1344 | { | |
e87c4f66 | 1345 | kcsan_mb(); |
67d1b0de MR |
1346 | instrument_atomic_read_write(v, sizeof(*v)); |
1347 | return arch_atomic_long_add_return(i, v); | |
1348 | } | |
1349 | ||
1350 | static __always_inline long | |
1351 | atomic_long_add_return_acquire(long i, atomic_long_t *v) | |
1352 | { | |
1353 | instrument_atomic_read_write(v, sizeof(*v)); | |
1354 | return arch_atomic_long_add_return_acquire(i, v); | |
1355 | } | |
1356 | ||
1357 | static __always_inline long | |
1358 | atomic_long_add_return_release(long i, atomic_long_t *v) | |
1359 | { | |
e87c4f66 | 1360 | kcsan_release(); |
67d1b0de MR |
1361 | instrument_atomic_read_write(v, sizeof(*v)); |
1362 | return arch_atomic_long_add_return_release(i, v); | |
1363 | } | |
1364 | ||
1365 | static __always_inline long | |
1366 | atomic_long_add_return_relaxed(long i, atomic_long_t *v) | |
1367 | { | |
1368 | instrument_atomic_read_write(v, sizeof(*v)); | |
1369 | return arch_atomic_long_add_return_relaxed(i, v); | |
1370 | } | |
1371 | ||
1372 | static __always_inline long | |
1373 | atomic_long_fetch_add(long i, atomic_long_t *v) | |
1374 | { | |
e87c4f66 | 1375 | kcsan_mb(); |
67d1b0de MR |
1376 | instrument_atomic_read_write(v, sizeof(*v)); |
1377 | return arch_atomic_long_fetch_add(i, v); | |
1378 | } | |
1379 | ||
1380 | static __always_inline long | |
1381 | atomic_long_fetch_add_acquire(long i, atomic_long_t *v) | |
1382 | { | |
1383 | instrument_atomic_read_write(v, sizeof(*v)); | |
1384 | return arch_atomic_long_fetch_add_acquire(i, v); | |
1385 | } | |
1386 | ||
1387 | static __always_inline long | |
1388 | atomic_long_fetch_add_release(long i, atomic_long_t *v) | |
1389 | { | |
e87c4f66 | 1390 | kcsan_release(); |
67d1b0de MR |
1391 | instrument_atomic_read_write(v, sizeof(*v)); |
1392 | return arch_atomic_long_fetch_add_release(i, v); | |
1393 | } | |
1394 | ||
1395 | static __always_inline long | |
1396 | atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) | |
1397 | { | |
1398 | instrument_atomic_read_write(v, sizeof(*v)); | |
1399 | return arch_atomic_long_fetch_add_relaxed(i, v); | |
1400 | } | |
1401 | ||
1402 | static __always_inline void | |
1403 | atomic_long_sub(long i, atomic_long_t *v) | |
1404 | { | |
1405 | instrument_atomic_read_write(v, sizeof(*v)); | |
1406 | arch_atomic_long_sub(i, v); | |
1407 | } | |
1408 | ||
1409 | static __always_inline long | |
1410 | atomic_long_sub_return(long i, atomic_long_t *v) | |
1411 | { | |
e87c4f66 | 1412 | kcsan_mb(); |
67d1b0de MR |
1413 | instrument_atomic_read_write(v, sizeof(*v)); |
1414 | return arch_atomic_long_sub_return(i, v); | |
1415 | } | |
1416 | ||
1417 | static __always_inline long | |
1418 | atomic_long_sub_return_acquire(long i, atomic_long_t *v) | |
1419 | { | |
1420 | instrument_atomic_read_write(v, sizeof(*v)); | |
1421 | return arch_atomic_long_sub_return_acquire(i, v); | |
1422 | } | |
1423 | ||
1424 | static __always_inline long | |
1425 | atomic_long_sub_return_release(long i, atomic_long_t *v) | |
1426 | { | |
e87c4f66 | 1427 | kcsan_release(); |
67d1b0de MR |
1428 | instrument_atomic_read_write(v, sizeof(*v)); |
1429 | return arch_atomic_long_sub_return_release(i, v); | |
1430 | } | |
1431 | ||
1432 | static __always_inline long | |
1433 | atomic_long_sub_return_relaxed(long i, atomic_long_t *v) | |
1434 | { | |
1435 | instrument_atomic_read_write(v, sizeof(*v)); | |
1436 | return arch_atomic_long_sub_return_relaxed(i, v); | |
1437 | } | |
1438 | ||
1439 | static __always_inline long | |
1440 | atomic_long_fetch_sub(long i, atomic_long_t *v) | |
1441 | { | |
e87c4f66 | 1442 | kcsan_mb(); |
67d1b0de MR |
1443 | instrument_atomic_read_write(v, sizeof(*v)); |
1444 | return arch_atomic_long_fetch_sub(i, v); | |
1445 | } | |
1446 | ||
1447 | static __always_inline long | |
1448 | atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) | |
1449 | { | |
1450 | instrument_atomic_read_write(v, sizeof(*v)); | |
1451 | return arch_atomic_long_fetch_sub_acquire(i, v); | |
1452 | } | |
1453 | ||
1454 | static __always_inline long | |
1455 | atomic_long_fetch_sub_release(long i, atomic_long_t *v) | |
1456 | { | |
e87c4f66 | 1457 | kcsan_release(); |
67d1b0de MR |
1458 | instrument_atomic_read_write(v, sizeof(*v)); |
1459 | return arch_atomic_long_fetch_sub_release(i, v); | |
1460 | } | |
1461 | ||
1462 | static __always_inline long | |
1463 | atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) | |
1464 | { | |
1465 | instrument_atomic_read_write(v, sizeof(*v)); | |
1466 | return arch_atomic_long_fetch_sub_relaxed(i, v); | |
1467 | } | |
1468 | ||
1469 | static __always_inline void | |
1470 | atomic_long_inc(atomic_long_t *v) | |
1471 | { | |
1472 | instrument_atomic_read_write(v, sizeof(*v)); | |
1473 | arch_atomic_long_inc(v); | |
1474 | } | |
1475 | ||
1476 | static __always_inline long | |
1477 | atomic_long_inc_return(atomic_long_t *v) | |
1478 | { | |
e87c4f66 | 1479 | kcsan_mb(); |
67d1b0de MR |
1480 | instrument_atomic_read_write(v, sizeof(*v)); |
1481 | return arch_atomic_long_inc_return(v); | |
1482 | } | |
1483 | ||
1484 | static __always_inline long | |
1485 | atomic_long_inc_return_acquire(atomic_long_t *v) | |
1486 | { | |
1487 | instrument_atomic_read_write(v, sizeof(*v)); | |
1488 | return arch_atomic_long_inc_return_acquire(v); | |
1489 | } | |
1490 | ||
1491 | static __always_inline long | |
1492 | atomic_long_inc_return_release(atomic_long_t *v) | |
1493 | { | |
e87c4f66 | 1494 | kcsan_release(); |
67d1b0de MR |
1495 | instrument_atomic_read_write(v, sizeof(*v)); |
1496 | return arch_atomic_long_inc_return_release(v); | |
1497 | } | |
1498 | ||
1499 | static __always_inline long | |
1500 | atomic_long_inc_return_relaxed(atomic_long_t *v) | |
1501 | { | |
1502 | instrument_atomic_read_write(v, sizeof(*v)); | |
1503 | return arch_atomic_long_inc_return_relaxed(v); | |
1504 | } | |
1505 | ||
1506 | static __always_inline long | |
1507 | atomic_long_fetch_inc(atomic_long_t *v) | |
1508 | { | |
e87c4f66 | 1509 | kcsan_mb(); |
67d1b0de MR |
1510 | instrument_atomic_read_write(v, sizeof(*v)); |
1511 | return arch_atomic_long_fetch_inc(v); | |
1512 | } | |
1513 | ||
1514 | static __always_inline long | |
1515 | atomic_long_fetch_inc_acquire(atomic_long_t *v) | |
1516 | { | |
1517 | instrument_atomic_read_write(v, sizeof(*v)); | |
1518 | return arch_atomic_long_fetch_inc_acquire(v); | |
1519 | } | |
1520 | ||
1521 | static __always_inline long | |
1522 | atomic_long_fetch_inc_release(atomic_long_t *v) | |
1523 | { | |
e87c4f66 | 1524 | kcsan_release(); |
67d1b0de MR |
1525 | instrument_atomic_read_write(v, sizeof(*v)); |
1526 | return arch_atomic_long_fetch_inc_release(v); | |
1527 | } | |
1528 | ||
1529 | static __always_inline long | |
1530 | atomic_long_fetch_inc_relaxed(atomic_long_t *v) | |
1531 | { | |
1532 | instrument_atomic_read_write(v, sizeof(*v)); | |
1533 | return arch_atomic_long_fetch_inc_relaxed(v); | |
1534 | } | |
1535 | ||
1536 | static __always_inline void | |
1537 | atomic_long_dec(atomic_long_t *v) | |
1538 | { | |
1539 | instrument_atomic_read_write(v, sizeof(*v)); | |
1540 | arch_atomic_long_dec(v); | |
1541 | } | |
1542 | ||
1543 | static __always_inline long | |
1544 | atomic_long_dec_return(atomic_long_t *v) | |
1545 | { | |
e87c4f66 | 1546 | kcsan_mb(); |
67d1b0de MR |
1547 | instrument_atomic_read_write(v, sizeof(*v)); |
1548 | return arch_atomic_long_dec_return(v); | |
1549 | } | |
1550 | ||
1551 | static __always_inline long | |
1552 | atomic_long_dec_return_acquire(atomic_long_t *v) | |
1553 | { | |
1554 | instrument_atomic_read_write(v, sizeof(*v)); | |
1555 | return arch_atomic_long_dec_return_acquire(v); | |
1556 | } | |
1557 | ||
1558 | static __always_inline long | |
1559 | atomic_long_dec_return_release(atomic_long_t *v) | |
1560 | { | |
e87c4f66 | 1561 | kcsan_release(); |
67d1b0de MR |
1562 | instrument_atomic_read_write(v, sizeof(*v)); |
1563 | return arch_atomic_long_dec_return_release(v); | |
1564 | } | |
1565 | ||
1566 | static __always_inline long | |
1567 | atomic_long_dec_return_relaxed(atomic_long_t *v) | |
1568 | { | |
1569 | instrument_atomic_read_write(v, sizeof(*v)); | |
1570 | return arch_atomic_long_dec_return_relaxed(v); | |
1571 | } | |
1572 | ||
1573 | static __always_inline long | |
1574 | atomic_long_fetch_dec(atomic_long_t *v) | |
1575 | { | |
e87c4f66 | 1576 | kcsan_mb(); |
67d1b0de MR |
1577 | instrument_atomic_read_write(v, sizeof(*v)); |
1578 | return arch_atomic_long_fetch_dec(v); | |
1579 | } | |
1580 | ||
1581 | static __always_inline long | |
1582 | atomic_long_fetch_dec_acquire(atomic_long_t *v) | |
1583 | { | |
1584 | instrument_atomic_read_write(v, sizeof(*v)); | |
1585 | return arch_atomic_long_fetch_dec_acquire(v); | |
1586 | } | |
1587 | ||
1588 | static __always_inline long | |
1589 | atomic_long_fetch_dec_release(atomic_long_t *v) | |
1590 | { | |
e87c4f66 | 1591 | kcsan_release(); |
67d1b0de MR |
1592 | instrument_atomic_read_write(v, sizeof(*v)); |
1593 | return arch_atomic_long_fetch_dec_release(v); | |
1594 | } | |
1595 | ||
1596 | static __always_inline long | |
1597 | atomic_long_fetch_dec_relaxed(atomic_long_t *v) | |
1598 | { | |
1599 | instrument_atomic_read_write(v, sizeof(*v)); | |
1600 | return arch_atomic_long_fetch_dec_relaxed(v); | |
1601 | } | |
1602 | ||
1603 | static __always_inline void | |
1604 | atomic_long_and(long i, atomic_long_t *v) | |
1605 | { | |
1606 | instrument_atomic_read_write(v, sizeof(*v)); | |
1607 | arch_atomic_long_and(i, v); | |
1608 | } | |
1609 | ||
1610 | static __always_inline long | |
1611 | atomic_long_fetch_and(long i, atomic_long_t *v) | |
1612 | { | |
e87c4f66 | 1613 | kcsan_mb(); |
67d1b0de MR |
1614 | instrument_atomic_read_write(v, sizeof(*v)); |
1615 | return arch_atomic_long_fetch_and(i, v); | |
1616 | } | |
1617 | ||
1618 | static __always_inline long | |
1619 | atomic_long_fetch_and_acquire(long i, atomic_long_t *v) | |
1620 | { | |
1621 | instrument_atomic_read_write(v, sizeof(*v)); | |
1622 | return arch_atomic_long_fetch_and_acquire(i, v); | |
1623 | } | |
1624 | ||
1625 | static __always_inline long | |
1626 | atomic_long_fetch_and_release(long i, atomic_long_t *v) | |
1627 | { | |
e87c4f66 | 1628 | kcsan_release(); |
67d1b0de MR |
1629 | instrument_atomic_read_write(v, sizeof(*v)); |
1630 | return arch_atomic_long_fetch_and_release(i, v); | |
1631 | } | |
1632 | ||
1633 | static __always_inline long | |
1634 | atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) | |
1635 | { | |
1636 | instrument_atomic_read_write(v, sizeof(*v)); | |
1637 | return arch_atomic_long_fetch_and_relaxed(i, v); | |
1638 | } | |
1639 | ||
1640 | static __always_inline void | |
1641 | atomic_long_andnot(long i, atomic_long_t *v) | |
1642 | { | |
1643 | instrument_atomic_read_write(v, sizeof(*v)); | |
1644 | arch_atomic_long_andnot(i, v); | |
1645 | } | |
1646 | ||
1647 | static __always_inline long | |
1648 | atomic_long_fetch_andnot(long i, atomic_long_t *v) | |
1649 | { | |
e87c4f66 | 1650 | kcsan_mb(); |
67d1b0de MR |
1651 | instrument_atomic_read_write(v, sizeof(*v)); |
1652 | return arch_atomic_long_fetch_andnot(i, v); | |
1653 | } | |
1654 | ||
1655 | static __always_inline long | |
1656 | atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) | |
1657 | { | |
1658 | instrument_atomic_read_write(v, sizeof(*v)); | |
1659 | return arch_atomic_long_fetch_andnot_acquire(i, v); | |
1660 | } | |
1661 | ||
1662 | static __always_inline long | |
1663 | atomic_long_fetch_andnot_release(long i, atomic_long_t *v) | |
1664 | { | |
e87c4f66 | 1665 | kcsan_release(); |
67d1b0de MR |
1666 | instrument_atomic_read_write(v, sizeof(*v)); |
1667 | return arch_atomic_long_fetch_andnot_release(i, v); | |
1668 | } | |
1669 | ||
1670 | static __always_inline long | |
1671 | atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) | |
1672 | { | |
1673 | instrument_atomic_read_write(v, sizeof(*v)); | |
1674 | return arch_atomic_long_fetch_andnot_relaxed(i, v); | |
1675 | } | |
1676 | ||
1677 | static __always_inline void | |
1678 | atomic_long_or(long i, atomic_long_t *v) | |
1679 | { | |
1680 | instrument_atomic_read_write(v, sizeof(*v)); | |
1681 | arch_atomic_long_or(i, v); | |
1682 | } | |
1683 | ||
1684 | static __always_inline long | |
1685 | atomic_long_fetch_or(long i, atomic_long_t *v) | |
1686 | { | |
e87c4f66 | 1687 | kcsan_mb(); |
67d1b0de MR |
1688 | instrument_atomic_read_write(v, sizeof(*v)); |
1689 | return arch_atomic_long_fetch_or(i, v); | |
1690 | } | |
1691 | ||
1692 | static __always_inline long | |
1693 | atomic_long_fetch_or_acquire(long i, atomic_long_t *v) | |
1694 | { | |
1695 | instrument_atomic_read_write(v, sizeof(*v)); | |
1696 | return arch_atomic_long_fetch_or_acquire(i, v); | |
1697 | } | |
1698 | ||
1699 | static __always_inline long | |
1700 | atomic_long_fetch_or_release(long i, atomic_long_t *v) | |
1701 | { | |
e87c4f66 | 1702 | kcsan_release(); |
67d1b0de MR |
1703 | instrument_atomic_read_write(v, sizeof(*v)); |
1704 | return arch_atomic_long_fetch_or_release(i, v); | |
1705 | } | |
1706 | ||
1707 | static __always_inline long | |
1708 | atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) | |
1709 | { | |
1710 | instrument_atomic_read_write(v, sizeof(*v)); | |
1711 | return arch_atomic_long_fetch_or_relaxed(i, v); | |
1712 | } | |
1713 | ||
1714 | static __always_inline void | |
1715 | atomic_long_xor(long i, atomic_long_t *v) | |
1716 | { | |
1717 | instrument_atomic_read_write(v, sizeof(*v)); | |
1718 | arch_atomic_long_xor(i, v); | |
1719 | } | |
1720 | ||
1721 | static __always_inline long | |
1722 | atomic_long_fetch_xor(long i, atomic_long_t *v) | |
1723 | { | |
e87c4f66 | 1724 | kcsan_mb(); |
67d1b0de MR |
1725 | instrument_atomic_read_write(v, sizeof(*v)); |
1726 | return arch_atomic_long_fetch_xor(i, v); | |
1727 | } | |
1728 | ||
1729 | static __always_inline long | |
1730 | atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) | |
1731 | { | |
1732 | instrument_atomic_read_write(v, sizeof(*v)); | |
1733 | return arch_atomic_long_fetch_xor_acquire(i, v); | |
1734 | } | |
1735 | ||
1736 | static __always_inline long | |
1737 | atomic_long_fetch_xor_release(long i, atomic_long_t *v) | |
1738 | { | |
e87c4f66 | 1739 | kcsan_release(); |
67d1b0de MR |
1740 | instrument_atomic_read_write(v, sizeof(*v)); |
1741 | return arch_atomic_long_fetch_xor_release(i, v); | |
1742 | } | |
1743 | ||
1744 | static __always_inline long | |
1745 | atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) | |
1746 | { | |
1747 | instrument_atomic_read_write(v, sizeof(*v)); | |
1748 | return arch_atomic_long_fetch_xor_relaxed(i, v); | |
1749 | } | |
1750 | ||
1751 | static __always_inline long | |
1752 | atomic_long_xchg(atomic_long_t *v, long i) | |
1753 | { | |
e87c4f66 | 1754 | kcsan_mb(); |
67d1b0de MR |
1755 | instrument_atomic_read_write(v, sizeof(*v)); |
1756 | return arch_atomic_long_xchg(v, i); | |
1757 | } | |
1758 | ||
1759 | static __always_inline long | |
1760 | atomic_long_xchg_acquire(atomic_long_t *v, long i) | |
1761 | { | |
1762 | instrument_atomic_read_write(v, sizeof(*v)); | |
1763 | return arch_atomic_long_xchg_acquire(v, i); | |
1764 | } | |
1765 | ||
1766 | static __always_inline long | |
1767 | atomic_long_xchg_release(atomic_long_t *v, long i) | |
1768 | { | |
e87c4f66 | 1769 | kcsan_release(); |
67d1b0de MR |
1770 | instrument_atomic_read_write(v, sizeof(*v)); |
1771 | return arch_atomic_long_xchg_release(v, i); | |
1772 | } | |
1773 | ||
1774 | static __always_inline long | |
1775 | atomic_long_xchg_relaxed(atomic_long_t *v, long i) | |
1776 | { | |
1777 | instrument_atomic_read_write(v, sizeof(*v)); | |
1778 | return arch_atomic_long_xchg_relaxed(v, i); | |
1779 | } | |
1780 | ||
1781 | static __always_inline long | |
1782 | atomic_long_cmpxchg(atomic_long_t *v, long old, long new) | |
1783 | { | |
e87c4f66 | 1784 | kcsan_mb(); |
67d1b0de MR |
1785 | instrument_atomic_read_write(v, sizeof(*v)); |
1786 | return arch_atomic_long_cmpxchg(v, old, new); | |
1787 | } | |
1788 | ||
1789 | static __always_inline long | |
1790 | atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) | |
1791 | { | |
1792 | instrument_atomic_read_write(v, sizeof(*v)); | |
1793 | return arch_atomic_long_cmpxchg_acquire(v, old, new); | |
1794 | } | |
1795 | ||
1796 | static __always_inline long | |
1797 | atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) | |
1798 | { | |
e87c4f66 | 1799 | kcsan_release(); |
67d1b0de MR |
1800 | instrument_atomic_read_write(v, sizeof(*v)); |
1801 | return arch_atomic_long_cmpxchg_release(v, old, new); | |
1802 | } | |
1803 | ||
1804 | static __always_inline long | |
1805 | atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) | |
1806 | { | |
1807 | instrument_atomic_read_write(v, sizeof(*v)); | |
1808 | return arch_atomic_long_cmpxchg_relaxed(v, old, new); | |
1809 | } | |
1810 | ||
1811 | static __always_inline bool | |
1812 | atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) | |
1813 | { | |
e87c4f66 | 1814 | kcsan_mb(); |
67d1b0de MR |
1815 | instrument_atomic_read_write(v, sizeof(*v)); |
1816 | instrument_atomic_read_write(old, sizeof(*old)); | |
1817 | return arch_atomic_long_try_cmpxchg(v, old, new); | |
1818 | } | |
1819 | ||
1820 | static __always_inline bool | |
1821 | atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) | |
1822 | { | |
1823 | instrument_atomic_read_write(v, sizeof(*v)); | |
1824 | instrument_atomic_read_write(old, sizeof(*old)); | |
1825 | return arch_atomic_long_try_cmpxchg_acquire(v, old, new); | |
1826 | } | |
1827 | ||
1828 | static __always_inline bool | |
1829 | atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) | |
1830 | { | |
e87c4f66 | 1831 | kcsan_release(); |
67d1b0de MR |
1832 | instrument_atomic_read_write(v, sizeof(*v)); |
1833 | instrument_atomic_read_write(old, sizeof(*old)); | |
1834 | return arch_atomic_long_try_cmpxchg_release(v, old, new); | |
1835 | } | |
1836 | ||
1837 | static __always_inline bool | |
1838 | atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) | |
1839 | { | |
1840 | instrument_atomic_read_write(v, sizeof(*v)); | |
1841 | instrument_atomic_read_write(old, sizeof(*old)); | |
1842 | return arch_atomic_long_try_cmpxchg_relaxed(v, old, new); | |
1843 | } | |
1844 | ||
1845 | static __always_inline bool | |
1846 | atomic_long_sub_and_test(long i, atomic_long_t *v) | |
1847 | { | |
e87c4f66 | 1848 | kcsan_mb(); |
67d1b0de MR |
1849 | instrument_atomic_read_write(v, sizeof(*v)); |
1850 | return arch_atomic_long_sub_and_test(i, v); | |
1851 | } | |
1852 | ||
1853 | static __always_inline bool | |
1854 | atomic_long_dec_and_test(atomic_long_t *v) | |
1855 | { | |
e87c4f66 | 1856 | kcsan_mb(); |
67d1b0de MR |
1857 | instrument_atomic_read_write(v, sizeof(*v)); |
1858 | return arch_atomic_long_dec_and_test(v); | |
1859 | } | |
1860 | ||
1861 | static __always_inline bool | |
1862 | atomic_long_inc_and_test(atomic_long_t *v) | |
1863 | { | |
e87c4f66 | 1864 | kcsan_mb(); |
67d1b0de MR |
1865 | instrument_atomic_read_write(v, sizeof(*v)); |
1866 | return arch_atomic_long_inc_and_test(v); | |
1867 | } | |
1868 | ||
1869 | static __always_inline bool | |
1870 | atomic_long_add_negative(long i, atomic_long_t *v) | |
1871 | { | |
e87c4f66 | 1872 | kcsan_mb(); |
67d1b0de MR |
1873 | instrument_atomic_read_write(v, sizeof(*v)); |
1874 | return arch_atomic_long_add_negative(i, v); | |
1875 | } | |
1876 | ||
e5ab9eff TG |
1877 | static __always_inline bool |
1878 | atomic_long_add_negative_acquire(long i, atomic_long_t *v) | |
1879 | { | |
1880 | instrument_atomic_read_write(v, sizeof(*v)); | |
1881 | return arch_atomic_long_add_negative_acquire(i, v); | |
1882 | } | |
1883 | ||
1884 | static __always_inline bool | |
1885 | atomic_long_add_negative_release(long i, atomic_long_t *v) | |
1886 | { | |
1887 | kcsan_release(); | |
1888 | instrument_atomic_read_write(v, sizeof(*v)); | |
1889 | return arch_atomic_long_add_negative_release(i, v); | |
1890 | } | |
1891 | ||
1892 | static __always_inline bool | |
1893 | atomic_long_add_negative_relaxed(long i, atomic_long_t *v) | |
1894 | { | |
1895 | instrument_atomic_read_write(v, sizeof(*v)); | |
1896 | return arch_atomic_long_add_negative_relaxed(i, v); | |
1897 | } | |
1898 | ||
67d1b0de MR |
1899 | static __always_inline long |
1900 | atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) | |
1901 | { | |
e87c4f66 | 1902 | kcsan_mb(); |
67d1b0de MR |
1903 | instrument_atomic_read_write(v, sizeof(*v)); |
1904 | return arch_atomic_long_fetch_add_unless(v, a, u); | |
1905 | } | |
1906 | ||
1907 | static __always_inline bool | |
1908 | atomic_long_add_unless(atomic_long_t *v, long a, long u) | |
1909 | { | |
e87c4f66 | 1910 | kcsan_mb(); |
67d1b0de MR |
1911 | instrument_atomic_read_write(v, sizeof(*v)); |
1912 | return arch_atomic_long_add_unless(v, a, u); | |
1913 | } | |
1914 | ||
1915 | static __always_inline bool | |
1916 | atomic_long_inc_not_zero(atomic_long_t *v) | |
1917 | { | |
e87c4f66 | 1918 | kcsan_mb(); |
67d1b0de MR |
1919 | instrument_atomic_read_write(v, sizeof(*v)); |
1920 | return arch_atomic_long_inc_not_zero(v); | |
1921 | } | |
1922 | ||
1923 | static __always_inline bool | |
1924 | atomic_long_inc_unless_negative(atomic_long_t *v) | |
1925 | { | |
e87c4f66 | 1926 | kcsan_mb(); |
67d1b0de MR |
1927 | instrument_atomic_read_write(v, sizeof(*v)); |
1928 | return arch_atomic_long_inc_unless_negative(v); | |
1929 | } | |
1930 | ||
1931 | static __always_inline bool | |
1932 | atomic_long_dec_unless_positive(atomic_long_t *v) | |
1933 | { | |
e87c4f66 | 1934 | kcsan_mb(); |
67d1b0de MR |
1935 | instrument_atomic_read_write(v, sizeof(*v)); |
1936 | return arch_atomic_long_dec_unless_positive(v); | |
1937 | } | |
1938 | ||
1939 | static __always_inline long | |
1940 | atomic_long_dec_if_positive(atomic_long_t *v) | |
1941 | { | |
e87c4f66 | 1942 | kcsan_mb(); |
67d1b0de MR |
1943 | instrument_atomic_read_write(v, sizeof(*v)); |
1944 | return arch_atomic_long_dec_if_positive(v); | |
1945 | } | |
1946 | ||
29f006fd PZ |
1947 | #define xchg(ptr, ...) \ |
1948 | ({ \ | |
1949 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 1950 | kcsan_mb(); \ |
ec570320 | 1951 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 1952 | arch_xchg(__ai_ptr, __VA_ARGS__); \ |
aa525d06 | 1953 | }) |
aa525d06 | 1954 | |
29f006fd PZ |
1955 | #define xchg_acquire(ptr, ...) \ |
1956 | ({ \ | |
1957 | typeof(ptr) __ai_ptr = (ptr); \ | |
ec570320 | 1958 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 1959 | arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ |
aa525d06 | 1960 | }) |
aa525d06 | 1961 | |
29f006fd PZ |
1962 | #define xchg_release(ptr, ...) \ |
1963 | ({ \ | |
1964 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 1965 | kcsan_release(); \ |
ec570320 | 1966 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 1967 | arch_xchg_release(__ai_ptr, __VA_ARGS__); \ |
aa525d06 | 1968 | }) |
aa525d06 | 1969 | |
29f006fd PZ |
1970 | #define xchg_relaxed(ptr, ...) \ |
1971 | ({ \ | |
1972 | typeof(ptr) __ai_ptr = (ptr); \ | |
ec570320 | 1973 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 1974 | arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ |
aa525d06 | 1975 | }) |
aa525d06 | 1976 | |
29f006fd PZ |
1977 | #define cmpxchg(ptr, ...) \ |
1978 | ({ \ | |
1979 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 1980 | kcsan_mb(); \ |
ec570320 | 1981 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 1982 | arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ |
f9881cc4 | 1983 | }) |
aa525d06 | 1984 | |
29f006fd PZ |
1985 | #define cmpxchg_acquire(ptr, ...) \ |
1986 | ({ \ | |
1987 | typeof(ptr) __ai_ptr = (ptr); \ | |
ec570320 | 1988 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 1989 | arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ |
aa525d06 | 1990 | }) |
aa525d06 | 1991 | |
29f006fd PZ |
1992 | #define cmpxchg_release(ptr, ...) \ |
1993 | ({ \ | |
1994 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 1995 | kcsan_release(); \ |
ec570320 | 1996 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 1997 | arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ |
aa525d06 | 1998 | }) |
aa525d06 | 1999 | |
29f006fd PZ |
2000 | #define cmpxchg_relaxed(ptr, ...) \ |
2001 | ({ \ | |
2002 | typeof(ptr) __ai_ptr = (ptr); \ | |
ec570320 | 2003 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 2004 | arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ |
aa525d06 | 2005 | }) |
f9881cc4 | 2006 | |
29f006fd PZ |
2007 | #define cmpxchg64(ptr, ...) \ |
2008 | ({ \ | |
2009 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 2010 | kcsan_mb(); \ |
ec570320 | 2011 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 2012 | arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ |
b06ed71a | 2013 | }) |
aa525d06 | 2014 | |
29f006fd PZ |
2015 | #define cmpxchg64_acquire(ptr, ...) \ |
2016 | ({ \ | |
2017 | typeof(ptr) __ai_ptr = (ptr); \ | |
ec570320 | 2018 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 2019 | arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ |
aa525d06 | 2020 | }) |
aa525d06 | 2021 | |
29f006fd PZ |
2022 | #define cmpxchg64_release(ptr, ...) \ |
2023 | ({ \ | |
2024 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 2025 | kcsan_release(); \ |
ec570320 | 2026 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 2027 | arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ |
aa525d06 | 2028 | }) |
b06ed71a | 2029 | |
29f006fd PZ |
2030 | #define cmpxchg64_relaxed(ptr, ...) \ |
2031 | ({ \ | |
2032 | typeof(ptr) __ai_ptr = (ptr); \ | |
ec570320 | 2033 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 2034 | arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ |
b06ed71a DV |
2035 | }) |
2036 | ||
29f006fd PZ |
2037 | #define try_cmpxchg(ptr, oldp, ...) \ |
2038 | ({ \ | |
2039 | typeof(ptr) __ai_ptr = (ptr); \ | |
2040 | typeof(oldp) __ai_oldp = (oldp); \ | |
e87c4f66 | 2041 | kcsan_mb(); \ |
ec570320 MR |
2042 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
2043 | instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
29f006fd PZ |
2044 | arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ |
2045 | }) | |
29f006fd | 2046 | |
29f006fd PZ |
2047 | #define try_cmpxchg_acquire(ptr, oldp, ...) \ |
2048 | ({ \ | |
2049 | typeof(ptr) __ai_ptr = (ptr); \ | |
2050 | typeof(oldp) __ai_oldp = (oldp); \ | |
ec570320 MR |
2051 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
2052 | instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
29f006fd PZ |
2053 | arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ |
2054 | }) | |
29f006fd | 2055 | |
29f006fd PZ |
2056 | #define try_cmpxchg_release(ptr, oldp, ...) \ |
2057 | ({ \ | |
2058 | typeof(ptr) __ai_ptr = (ptr); \ | |
2059 | typeof(oldp) __ai_oldp = (oldp); \ | |
e87c4f66 | 2060 | kcsan_release(); \ |
ec570320 MR |
2061 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
2062 | instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
29f006fd PZ |
2063 | arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ |
2064 | }) | |
29f006fd | 2065 | |
29f006fd PZ |
2066 | #define try_cmpxchg_relaxed(ptr, oldp, ...) \ |
2067 | ({ \ | |
2068 | typeof(ptr) __ai_ptr = (ptr); \ | |
2069 | typeof(oldp) __ai_oldp = (oldp); \ | |
ec570320 MR |
2070 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
2071 | instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
29f006fd PZ |
2072 | arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ |
2073 | }) | |
29f006fd | 2074 | |
0aa7be05 UB |
2075 | #define try_cmpxchg64(ptr, oldp, ...) \ |
2076 | ({ \ | |
2077 | typeof(ptr) __ai_ptr = (ptr); \ | |
2078 | typeof(oldp) __ai_oldp = (oldp); \ | |
2079 | kcsan_mb(); \ | |
ec570320 MR |
2080 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
2081 | instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
0aa7be05 UB |
2082 | arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \ |
2083 | }) | |
2084 | ||
2085 | #define try_cmpxchg64_acquire(ptr, oldp, ...) \ | |
2086 | ({ \ | |
2087 | typeof(ptr) __ai_ptr = (ptr); \ | |
2088 | typeof(oldp) __ai_oldp = (oldp); \ | |
ec570320 MR |
2089 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
2090 | instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
0aa7be05 UB |
2091 | arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ |
2092 | }) | |
2093 | ||
2094 | #define try_cmpxchg64_release(ptr, oldp, ...) \ | |
2095 | ({ \ | |
2096 | typeof(ptr) __ai_ptr = (ptr); \ | |
2097 | typeof(oldp) __ai_oldp = (oldp); \ | |
2098 | kcsan_release(); \ | |
ec570320 MR |
2099 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
2100 | instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
0aa7be05 UB |
2101 | arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ |
2102 | }) | |
2103 | ||
2104 | #define try_cmpxchg64_relaxed(ptr, oldp, ...) \ | |
2105 | ({ \ | |
2106 | typeof(ptr) __ai_ptr = (ptr); \ | |
2107 | typeof(oldp) __ai_oldp = (oldp); \ | |
ec570320 MR |
2108 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
2109 | instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
0aa7be05 UB |
2110 | arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ |
2111 | }) | |
2112 | ||
29f006fd PZ |
2113 | #define cmpxchg_local(ptr, ...) \ |
2114 | ({ \ | |
2115 | typeof(ptr) __ai_ptr = (ptr); \ | |
ec570320 | 2116 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 2117 | arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ |
b06ed71a DV |
2118 | }) |
2119 | ||
29f006fd PZ |
2120 | #define cmpxchg64_local(ptr, ...) \ |
2121 | ({ \ | |
2122 | typeof(ptr) __ai_ptr = (ptr); \ | |
ec570320 | 2123 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 2124 | arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ |
b06ed71a DV |
2125 | }) |
2126 | ||
29f006fd PZ |
2127 | #define sync_cmpxchg(ptr, ...) \ |
2128 | ({ \ | |
2129 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 2130 | kcsan_mb(); \ |
ec570320 | 2131 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
29f006fd | 2132 | arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ |
b06ed71a DV |
2133 | }) |
2134 | ||
e6ce9d74 UB |
2135 | #define try_cmpxchg_local(ptr, oldp, ...) \ |
2136 | ({ \ | |
2137 | typeof(ptr) __ai_ptr = (ptr); \ | |
2138 | typeof(oldp) __ai_oldp = (oldp); \ | |
ec570320 MR |
2139 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
2140 | instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
e6ce9d74 UB |
2141 | arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ |
2142 | }) | |
2143 | ||
2144 | #define try_cmpxchg64_local(ptr, oldp, ...) \ | |
2145 | ({ \ | |
2146 | typeof(ptr) __ai_ptr = (ptr); \ | |
2147 | typeof(oldp) __ai_oldp = (oldp); \ | |
ec570320 MR |
2148 | instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ |
2149 | instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ | |
e6ce9d74 UB |
2150 | arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ |
2151 | }) | |
2152 | ||
29f006fd PZ |
2153 | #define cmpxchg_double(ptr, ...) \ |
2154 | ({ \ | |
2155 | typeof(ptr) __ai_ptr = (ptr); \ | |
e87c4f66 | 2156 | kcsan_mb(); \ |
ec570320 | 2157 | instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ |
29f006fd | 2158 | arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ |
b06ed71a DV |
2159 | }) |
2160 | ||
aa525d06 | 2161 | |
29f006fd PZ |
2162 | #define cmpxchg_double_local(ptr, ...) \ |
2163 | ({ \ | |
2164 | typeof(ptr) __ai_ptr = (ptr); \ | |
ec570320 | 2165 | instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ |
29f006fd | 2166 | arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ |
b06ed71a DV |
2167 | }) |
2168 | ||
e3d18cee | 2169 | #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ |
ec570320 | 2170 | // 6b513a42e1a1b5962532a019b7fc91eaa044ad5e |