1 // SPDX-License-Identifier: GPL-2.0
3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 // DO NOT MODIFY THIS FILE DIRECTLY
7 * This file provoides atomic operations with explicit instrumentation (e.g.
8 * KASAN, KCSAN), which should be used unless it is necessary to avoid
9 * instrumentation. Where it is necessary to aovid instrumenation, the
10 * raw_atomic*() operations should be used.
12 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
13 #define _LINUX_ATOMIC_INSTRUMENTED_H
15 #include <linux/build_bug.h>
16 #include <linux/compiler.h>
17 #include <linux/instrumented.h>
20 * atomic_read() - atomic load with relaxed ordering
21 * @v: pointer to atomic_t
23 * Atomically loads the value of @v with relaxed ordering.
25 * Unsafe to use in noinstr code; use raw_atomic_read() there.
27 * Return: The value loaded from @v.
29 static __always_inline int
30 atomic_read(const atomic_t *v)
32 instrument_atomic_read(v, sizeof(*v));
33 return raw_atomic_read(v);
37 * atomic_read_acquire() - atomic load with acquire ordering
38 * @v: pointer to atomic_t
40 * Atomically loads the value of @v with acquire ordering.
42 * Unsafe to use in noinstr code; use raw_atomic_read_acquire() there.
44 * Return: The value loaded from @v.
46 static __always_inline int
47 atomic_read_acquire(const atomic_t *v)
49 instrument_atomic_read(v, sizeof(*v));
50 return raw_atomic_read_acquire(v);
54 * atomic_set() - atomic set with relaxed ordering
55 * @v: pointer to atomic_t
56 * @i: int value to assign
58 * Atomically sets @v to @i with relaxed ordering.
60 * Unsafe to use in noinstr code; use raw_atomic_set() there.
64 static __always_inline void
65 atomic_set(atomic_t *v, int i)
67 instrument_atomic_write(v, sizeof(*v));
72 * atomic_set_release() - atomic set with release ordering
73 * @v: pointer to atomic_t
74 * @i: int value to assign
76 * Atomically sets @v to @i with release ordering.
78 * Unsafe to use in noinstr code; use raw_atomic_set_release() there.
82 static __always_inline void
83 atomic_set_release(atomic_t *v, int i)
86 instrument_atomic_write(v, sizeof(*v));
87 raw_atomic_set_release(v, i);
91 * atomic_add() - atomic add with relaxed ordering
92 * @i: int value to add
93 * @v: pointer to atomic_t
95 * Atomically updates @v to (@v + @i) with relaxed ordering.
97 * Unsafe to use in noinstr code; use raw_atomic_add() there.
101 static __always_inline void
102 atomic_add(int i, atomic_t *v)
104 instrument_atomic_read_write(v, sizeof(*v));
105 raw_atomic_add(i, v);
109 * atomic_add_return() - atomic add with full ordering
110 * @i: int value to add
111 * @v: pointer to atomic_t
113 * Atomically updates @v to (@v + @i) with full ordering.
115 * Unsafe to use in noinstr code; use raw_atomic_add_return() there.
117 * Return: The updated value of @v.
119 static __always_inline int
120 atomic_add_return(int i, atomic_t *v)
123 instrument_atomic_read_write(v, sizeof(*v));
124 return raw_atomic_add_return(i, v);
128 * atomic_add_return_acquire() - atomic add with acquire ordering
129 * @i: int value to add
130 * @v: pointer to atomic_t
132 * Atomically updates @v to (@v + @i) with acquire ordering.
134 * Unsafe to use in noinstr code; use raw_atomic_add_return_acquire() there.
136 * Return: The updated value of @v.
138 static __always_inline int
139 atomic_add_return_acquire(int i, atomic_t *v)
141 instrument_atomic_read_write(v, sizeof(*v));
142 return raw_atomic_add_return_acquire(i, v);
146 * atomic_add_return_release() - atomic add with release ordering
147 * @i: int value to add
148 * @v: pointer to atomic_t
150 * Atomically updates @v to (@v + @i) with release ordering.
152 * Unsafe to use in noinstr code; use raw_atomic_add_return_release() there.
154 * Return: The updated value of @v.
156 static __always_inline int
157 atomic_add_return_release(int i, atomic_t *v)
160 instrument_atomic_read_write(v, sizeof(*v));
161 return raw_atomic_add_return_release(i, v);
165 * atomic_add_return_relaxed() - atomic add with relaxed ordering
166 * @i: int value to add
167 * @v: pointer to atomic_t
169 * Atomically updates @v to (@v + @i) with relaxed ordering.
171 * Unsafe to use in noinstr code; use raw_atomic_add_return_relaxed() there.
173 * Return: The updated value of @v.
175 static __always_inline int
176 atomic_add_return_relaxed(int i, atomic_t *v)
178 instrument_atomic_read_write(v, sizeof(*v));
179 return raw_atomic_add_return_relaxed(i, v);
183 * atomic_fetch_add() - atomic add with full ordering
184 * @i: int value to add
185 * @v: pointer to atomic_t
187 * Atomically updates @v to (@v + @i) with full ordering.
189 * Unsafe to use in noinstr code; use raw_atomic_fetch_add() there.
191 * Return: The original value of @v.
193 static __always_inline int
194 atomic_fetch_add(int i, atomic_t *v)
197 instrument_atomic_read_write(v, sizeof(*v));
198 return raw_atomic_fetch_add(i, v);
202 * atomic_fetch_add_acquire() - atomic add with acquire ordering
203 * @i: int value to add
204 * @v: pointer to atomic_t
206 * Atomically updates @v to (@v + @i) with acquire ordering.
208 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_acquire() there.
210 * Return: The original value of @v.
212 static __always_inline int
213 atomic_fetch_add_acquire(int i, atomic_t *v)
215 instrument_atomic_read_write(v, sizeof(*v));
216 return raw_atomic_fetch_add_acquire(i, v);
220 * atomic_fetch_add_release() - atomic add with release ordering
221 * @i: int value to add
222 * @v: pointer to atomic_t
224 * Atomically updates @v to (@v + @i) with release ordering.
226 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_release() there.
228 * Return: The original value of @v.
230 static __always_inline int
231 atomic_fetch_add_release(int i, atomic_t *v)
234 instrument_atomic_read_write(v, sizeof(*v));
235 return raw_atomic_fetch_add_release(i, v);
239 * atomic_fetch_add_relaxed() - atomic add with relaxed ordering
240 * @i: int value to add
241 * @v: pointer to atomic_t
243 * Atomically updates @v to (@v + @i) with relaxed ordering.
245 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_relaxed() there.
247 * Return: The original value of @v.
249 static __always_inline int
250 atomic_fetch_add_relaxed(int i, atomic_t *v)
252 instrument_atomic_read_write(v, sizeof(*v));
253 return raw_atomic_fetch_add_relaxed(i, v);
257 * atomic_sub() - atomic subtract with relaxed ordering
258 * @i: int value to subtract
259 * @v: pointer to atomic_t
261 * Atomically updates @v to (@v - @i) with relaxed ordering.
263 * Unsafe to use in noinstr code; use raw_atomic_sub() there.
267 static __always_inline void
268 atomic_sub(int i, atomic_t *v)
270 instrument_atomic_read_write(v, sizeof(*v));
271 raw_atomic_sub(i, v);
275 * atomic_sub_return() - atomic subtract with full ordering
276 * @i: int value to subtract
277 * @v: pointer to atomic_t
279 * Atomically updates @v to (@v - @i) with full ordering.
281 * Unsafe to use in noinstr code; use raw_atomic_sub_return() there.
283 * Return: The updated value of @v.
285 static __always_inline int
286 atomic_sub_return(int i, atomic_t *v)
289 instrument_atomic_read_write(v, sizeof(*v));
290 return raw_atomic_sub_return(i, v);
294 * atomic_sub_return_acquire() - atomic subtract with acquire ordering
295 * @i: int value to subtract
296 * @v: pointer to atomic_t
298 * Atomically updates @v to (@v - @i) with acquire ordering.
300 * Unsafe to use in noinstr code; use raw_atomic_sub_return_acquire() there.
302 * Return: The updated value of @v.
304 static __always_inline int
305 atomic_sub_return_acquire(int i, atomic_t *v)
307 instrument_atomic_read_write(v, sizeof(*v));
308 return raw_atomic_sub_return_acquire(i, v);
312 * atomic_sub_return_release() - atomic subtract with release ordering
313 * @i: int value to subtract
314 * @v: pointer to atomic_t
316 * Atomically updates @v to (@v - @i) with release ordering.
318 * Unsafe to use in noinstr code; use raw_atomic_sub_return_release() there.
320 * Return: The updated value of @v.
322 static __always_inline int
323 atomic_sub_return_release(int i, atomic_t *v)
326 instrument_atomic_read_write(v, sizeof(*v));
327 return raw_atomic_sub_return_release(i, v);
331 * atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
332 * @i: int value to subtract
333 * @v: pointer to atomic_t
335 * Atomically updates @v to (@v - @i) with relaxed ordering.
337 * Unsafe to use in noinstr code; use raw_atomic_sub_return_relaxed() there.
339 * Return: The updated value of @v.
341 static __always_inline int
342 atomic_sub_return_relaxed(int i, atomic_t *v)
344 instrument_atomic_read_write(v, sizeof(*v));
345 return raw_atomic_sub_return_relaxed(i, v);
349 * atomic_fetch_sub() - atomic subtract with full ordering
350 * @i: int value to subtract
351 * @v: pointer to atomic_t
353 * Atomically updates @v to (@v - @i) with full ordering.
355 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub() there.
357 * Return: The original value of @v.
359 static __always_inline int
360 atomic_fetch_sub(int i, atomic_t *v)
363 instrument_atomic_read_write(v, sizeof(*v));
364 return raw_atomic_fetch_sub(i, v);
368 * atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
369 * @i: int value to subtract
370 * @v: pointer to atomic_t
372 * Atomically updates @v to (@v - @i) with acquire ordering.
374 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_acquire() there.
376 * Return: The original value of @v.
378 static __always_inline int
379 atomic_fetch_sub_acquire(int i, atomic_t *v)
381 instrument_atomic_read_write(v, sizeof(*v));
382 return raw_atomic_fetch_sub_acquire(i, v);
386 * atomic_fetch_sub_release() - atomic subtract with release ordering
387 * @i: int value to subtract
388 * @v: pointer to atomic_t
390 * Atomically updates @v to (@v - @i) with release ordering.
392 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_release() there.
394 * Return: The original value of @v.
396 static __always_inline int
397 atomic_fetch_sub_release(int i, atomic_t *v)
400 instrument_atomic_read_write(v, sizeof(*v));
401 return raw_atomic_fetch_sub_release(i, v);
405 * atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
406 * @i: int value to subtract
407 * @v: pointer to atomic_t
409 * Atomically updates @v to (@v - @i) with relaxed ordering.
411 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_relaxed() there.
413 * Return: The original value of @v.
415 static __always_inline int
416 atomic_fetch_sub_relaxed(int i, atomic_t *v)
418 instrument_atomic_read_write(v, sizeof(*v));
419 return raw_atomic_fetch_sub_relaxed(i, v);
423 * atomic_inc() - atomic increment with relaxed ordering
424 * @v: pointer to atomic_t
426 * Atomically updates @v to (@v + 1) with relaxed ordering.
428 * Unsafe to use in noinstr code; use raw_atomic_inc() there.
432 static __always_inline void
433 atomic_inc(atomic_t *v)
435 instrument_atomic_read_write(v, sizeof(*v));
440 * atomic_inc_return() - atomic increment with full ordering
441 * @v: pointer to atomic_t
443 * Atomically updates @v to (@v + 1) with full ordering.
445 * Unsafe to use in noinstr code; use raw_atomic_inc_return() there.
447 * Return: The updated value of @v.
449 static __always_inline int
450 atomic_inc_return(atomic_t *v)
453 instrument_atomic_read_write(v, sizeof(*v));
454 return raw_atomic_inc_return(v);
458 * atomic_inc_return_acquire() - atomic increment with acquire ordering
459 * @v: pointer to atomic_t
461 * Atomically updates @v to (@v + 1) with acquire ordering.
463 * Unsafe to use in noinstr code; use raw_atomic_inc_return_acquire() there.
465 * Return: The updated value of @v.
467 static __always_inline int
468 atomic_inc_return_acquire(atomic_t *v)
470 instrument_atomic_read_write(v, sizeof(*v));
471 return raw_atomic_inc_return_acquire(v);
475 * atomic_inc_return_release() - atomic increment with release ordering
476 * @v: pointer to atomic_t
478 * Atomically updates @v to (@v + 1) with release ordering.
480 * Unsafe to use in noinstr code; use raw_atomic_inc_return_release() there.
482 * Return: The updated value of @v.
484 static __always_inline int
485 atomic_inc_return_release(atomic_t *v)
488 instrument_atomic_read_write(v, sizeof(*v));
489 return raw_atomic_inc_return_release(v);
493 * atomic_inc_return_relaxed() - atomic increment with relaxed ordering
494 * @v: pointer to atomic_t
496 * Atomically updates @v to (@v + 1) with relaxed ordering.
498 * Unsafe to use in noinstr code; use raw_atomic_inc_return_relaxed() there.
500 * Return: The updated value of @v.
502 static __always_inline int
503 atomic_inc_return_relaxed(atomic_t *v)
505 instrument_atomic_read_write(v, sizeof(*v));
506 return raw_atomic_inc_return_relaxed(v);
510 * atomic_fetch_inc() - atomic increment with full ordering
511 * @v: pointer to atomic_t
513 * Atomically updates @v to (@v + 1) with full ordering.
515 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc() there.
517 * Return: The original value of @v.
519 static __always_inline int
520 atomic_fetch_inc(atomic_t *v)
523 instrument_atomic_read_write(v, sizeof(*v));
524 return raw_atomic_fetch_inc(v);
528 * atomic_fetch_inc_acquire() - atomic increment with acquire ordering
529 * @v: pointer to atomic_t
531 * Atomically updates @v to (@v + 1) with acquire ordering.
533 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_acquire() there.
535 * Return: The original value of @v.
537 static __always_inline int
538 atomic_fetch_inc_acquire(atomic_t *v)
540 instrument_atomic_read_write(v, sizeof(*v));
541 return raw_atomic_fetch_inc_acquire(v);
545 * atomic_fetch_inc_release() - atomic increment with release ordering
546 * @v: pointer to atomic_t
548 * Atomically updates @v to (@v + 1) with release ordering.
550 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_release() there.
552 * Return: The original value of @v.
554 static __always_inline int
555 atomic_fetch_inc_release(atomic_t *v)
558 instrument_atomic_read_write(v, sizeof(*v));
559 return raw_atomic_fetch_inc_release(v);
563 * atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
564 * @v: pointer to atomic_t
566 * Atomically updates @v to (@v + 1) with relaxed ordering.
568 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_relaxed() there.
570 * Return: The original value of @v.
572 static __always_inline int
573 atomic_fetch_inc_relaxed(atomic_t *v)
575 instrument_atomic_read_write(v, sizeof(*v));
576 return raw_atomic_fetch_inc_relaxed(v);
580 * atomic_dec() - atomic decrement with relaxed ordering
581 * @v: pointer to atomic_t
583 * Atomically updates @v to (@v - 1) with relaxed ordering.
585 * Unsafe to use in noinstr code; use raw_atomic_dec() there.
589 static __always_inline void
590 atomic_dec(atomic_t *v)
592 instrument_atomic_read_write(v, sizeof(*v));
597 * atomic_dec_return() - atomic decrement with full ordering
598 * @v: pointer to atomic_t
600 * Atomically updates @v to (@v - 1) with full ordering.
602 * Unsafe to use in noinstr code; use raw_atomic_dec_return() there.
604 * Return: The updated value of @v.
606 static __always_inline int
607 atomic_dec_return(atomic_t *v)
610 instrument_atomic_read_write(v, sizeof(*v));
611 return raw_atomic_dec_return(v);
615 * atomic_dec_return_acquire() - atomic decrement with acquire ordering
616 * @v: pointer to atomic_t
618 * Atomically updates @v to (@v - 1) with acquire ordering.
620 * Unsafe to use in noinstr code; use raw_atomic_dec_return_acquire() there.
622 * Return: The updated value of @v.
624 static __always_inline int
625 atomic_dec_return_acquire(atomic_t *v)
627 instrument_atomic_read_write(v, sizeof(*v));
628 return raw_atomic_dec_return_acquire(v);
632 * atomic_dec_return_release() - atomic decrement with release ordering
633 * @v: pointer to atomic_t
635 * Atomically updates @v to (@v - 1) with release ordering.
637 * Unsafe to use in noinstr code; use raw_atomic_dec_return_release() there.
639 * Return: The updated value of @v.
641 static __always_inline int
642 atomic_dec_return_release(atomic_t *v)
645 instrument_atomic_read_write(v, sizeof(*v));
646 return raw_atomic_dec_return_release(v);
650 * atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
651 * @v: pointer to atomic_t
653 * Atomically updates @v to (@v - 1) with relaxed ordering.
655 * Unsafe to use in noinstr code; use raw_atomic_dec_return_relaxed() there.
657 * Return: The updated value of @v.
659 static __always_inline int
660 atomic_dec_return_relaxed(atomic_t *v)
662 instrument_atomic_read_write(v, sizeof(*v));
663 return raw_atomic_dec_return_relaxed(v);
667 * atomic_fetch_dec() - atomic decrement with full ordering
668 * @v: pointer to atomic_t
670 * Atomically updates @v to (@v - 1) with full ordering.
672 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec() there.
674 * Return: The original value of @v.
676 static __always_inline int
677 atomic_fetch_dec(atomic_t *v)
680 instrument_atomic_read_write(v, sizeof(*v));
681 return raw_atomic_fetch_dec(v);
685 * atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
686 * @v: pointer to atomic_t
688 * Atomically updates @v to (@v - 1) with acquire ordering.
690 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_acquire() there.
692 * Return: The original value of @v.
694 static __always_inline int
695 atomic_fetch_dec_acquire(atomic_t *v)
697 instrument_atomic_read_write(v, sizeof(*v));
698 return raw_atomic_fetch_dec_acquire(v);
702 * atomic_fetch_dec_release() - atomic decrement with release ordering
703 * @v: pointer to atomic_t
705 * Atomically updates @v to (@v - 1) with release ordering.
707 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_release() there.
709 * Return: The original value of @v.
711 static __always_inline int
712 atomic_fetch_dec_release(atomic_t *v)
715 instrument_atomic_read_write(v, sizeof(*v));
716 return raw_atomic_fetch_dec_release(v);
720 * atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
721 * @v: pointer to atomic_t
723 * Atomically updates @v to (@v - 1) with relaxed ordering.
725 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_relaxed() there.
727 * Return: The original value of @v.
729 static __always_inline int
730 atomic_fetch_dec_relaxed(atomic_t *v)
732 instrument_atomic_read_write(v, sizeof(*v));
733 return raw_atomic_fetch_dec_relaxed(v);
737 * atomic_and() - atomic bitwise AND with relaxed ordering
739 * @v: pointer to atomic_t
741 * Atomically updates @v to (@v & @i) with relaxed ordering.
743 * Unsafe to use in noinstr code; use raw_atomic_and() there.
747 static __always_inline void
748 atomic_and(int i, atomic_t *v)
750 instrument_atomic_read_write(v, sizeof(*v));
751 raw_atomic_and(i, v);
755 * atomic_fetch_and() - atomic bitwise AND with full ordering
757 * @v: pointer to atomic_t
759 * Atomically updates @v to (@v & @i) with full ordering.
761 * Unsafe to use in noinstr code; use raw_atomic_fetch_and() there.
763 * Return: The original value of @v.
765 static __always_inline int
766 atomic_fetch_and(int i, atomic_t *v)
769 instrument_atomic_read_write(v, sizeof(*v));
770 return raw_atomic_fetch_and(i, v);
774 * atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
776 * @v: pointer to atomic_t
778 * Atomically updates @v to (@v & @i) with acquire ordering.
780 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_acquire() there.
782 * Return: The original value of @v.
784 static __always_inline int
785 atomic_fetch_and_acquire(int i, atomic_t *v)
787 instrument_atomic_read_write(v, sizeof(*v));
788 return raw_atomic_fetch_and_acquire(i, v);
792 * atomic_fetch_and_release() - atomic bitwise AND with release ordering
794 * @v: pointer to atomic_t
796 * Atomically updates @v to (@v & @i) with release ordering.
798 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_release() there.
800 * Return: The original value of @v.
802 static __always_inline int
803 atomic_fetch_and_release(int i, atomic_t *v)
806 instrument_atomic_read_write(v, sizeof(*v));
807 return raw_atomic_fetch_and_release(i, v);
811 * atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
813 * @v: pointer to atomic_t
815 * Atomically updates @v to (@v & @i) with relaxed ordering.
817 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_relaxed() there.
819 * Return: The original value of @v.
821 static __always_inline int
822 atomic_fetch_and_relaxed(int i, atomic_t *v)
824 instrument_atomic_read_write(v, sizeof(*v));
825 return raw_atomic_fetch_and_relaxed(i, v);
829 * atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
831 * @v: pointer to atomic_t
833 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
835 * Unsafe to use in noinstr code; use raw_atomic_andnot() there.
839 static __always_inline void
840 atomic_andnot(int i, atomic_t *v)
842 instrument_atomic_read_write(v, sizeof(*v));
843 raw_atomic_andnot(i, v);
847 * atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
849 * @v: pointer to atomic_t
851 * Atomically updates @v to (@v & ~@i) with full ordering.
853 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot() there.
855 * Return: The original value of @v.
857 static __always_inline int
858 atomic_fetch_andnot(int i, atomic_t *v)
861 instrument_atomic_read_write(v, sizeof(*v));
862 return raw_atomic_fetch_andnot(i, v);
866 * atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
868 * @v: pointer to atomic_t
870 * Atomically updates @v to (@v & ~@i) with acquire ordering.
872 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_acquire() there.
874 * Return: The original value of @v.
876 static __always_inline int
877 atomic_fetch_andnot_acquire(int i, atomic_t *v)
879 instrument_atomic_read_write(v, sizeof(*v));
880 return raw_atomic_fetch_andnot_acquire(i, v);
884 * atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
886 * @v: pointer to atomic_t
888 * Atomically updates @v to (@v & ~@i) with release ordering.
890 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_release() there.
892 * Return: The original value of @v.
894 static __always_inline int
895 atomic_fetch_andnot_release(int i, atomic_t *v)
898 instrument_atomic_read_write(v, sizeof(*v));
899 return raw_atomic_fetch_andnot_release(i, v);
903 * atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
905 * @v: pointer to atomic_t
907 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
909 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_relaxed() there.
911 * Return: The original value of @v.
913 static __always_inline int
914 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
916 instrument_atomic_read_write(v, sizeof(*v));
917 return raw_atomic_fetch_andnot_relaxed(i, v);
921 * atomic_or() - atomic bitwise OR with relaxed ordering
923 * @v: pointer to atomic_t
925 * Atomically updates @v to (@v | @i) with relaxed ordering.
927 * Unsafe to use in noinstr code; use raw_atomic_or() there.
931 static __always_inline void
932 atomic_or(int i, atomic_t *v)
934 instrument_atomic_read_write(v, sizeof(*v));
939 * atomic_fetch_or() - atomic bitwise OR with full ordering
941 * @v: pointer to atomic_t
943 * Atomically updates @v to (@v | @i) with full ordering.
945 * Unsafe to use in noinstr code; use raw_atomic_fetch_or() there.
947 * Return: The original value of @v.
949 static __always_inline int
950 atomic_fetch_or(int i, atomic_t *v)
953 instrument_atomic_read_write(v, sizeof(*v));
954 return raw_atomic_fetch_or(i, v);
958 * atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
960 * @v: pointer to atomic_t
962 * Atomically updates @v to (@v | @i) with acquire ordering.
964 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_acquire() there.
966 * Return: The original value of @v.
968 static __always_inline int
969 atomic_fetch_or_acquire(int i, atomic_t *v)
971 instrument_atomic_read_write(v, sizeof(*v));
972 return raw_atomic_fetch_or_acquire(i, v);
976 * atomic_fetch_or_release() - atomic bitwise OR with release ordering
978 * @v: pointer to atomic_t
980 * Atomically updates @v to (@v | @i) with release ordering.
982 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_release() there.
984 * Return: The original value of @v.
986 static __always_inline int
987 atomic_fetch_or_release(int i, atomic_t *v)
990 instrument_atomic_read_write(v, sizeof(*v));
991 return raw_atomic_fetch_or_release(i, v);
995 * atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
997 * @v: pointer to atomic_t
999 * Atomically updates @v to (@v | @i) with relaxed ordering.
1001 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_relaxed() there.
1003 * Return: The original value of @v.
1005 static __always_inline int
1006 atomic_fetch_or_relaxed(int i, atomic_t *v)
1008 instrument_atomic_read_write(v, sizeof(*v));
1009 return raw_atomic_fetch_or_relaxed(i, v);
1013 * atomic_xor() - atomic bitwise XOR with relaxed ordering
1015 * @v: pointer to atomic_t
1017 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1019 * Unsafe to use in noinstr code; use raw_atomic_xor() there.
1023 static __always_inline void
1024 atomic_xor(int i, atomic_t *v)
1026 instrument_atomic_read_write(v, sizeof(*v));
1027 raw_atomic_xor(i, v);
1031 * atomic_fetch_xor() - atomic bitwise XOR with full ordering
1033 * @v: pointer to atomic_t
1035 * Atomically updates @v to (@v ^ @i) with full ordering.
1037 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor() there.
1039 * Return: The original value of @v.
1041 static __always_inline int
1042 atomic_fetch_xor(int i, atomic_t *v)
1045 instrument_atomic_read_write(v, sizeof(*v));
1046 return raw_atomic_fetch_xor(i, v);
1050 * atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
1052 * @v: pointer to atomic_t
1054 * Atomically updates @v to (@v ^ @i) with acquire ordering.
1056 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_acquire() there.
1058 * Return: The original value of @v.
1060 static __always_inline int
1061 atomic_fetch_xor_acquire(int i, atomic_t *v)
1063 instrument_atomic_read_write(v, sizeof(*v));
1064 return raw_atomic_fetch_xor_acquire(i, v);
1068 * atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
1070 * @v: pointer to atomic_t
1072 * Atomically updates @v to (@v ^ @i) with release ordering.
1074 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_release() there.
1076 * Return: The original value of @v.
1078 static __always_inline int
1079 atomic_fetch_xor_release(int i, atomic_t *v)
1082 instrument_atomic_read_write(v, sizeof(*v));
1083 return raw_atomic_fetch_xor_release(i, v);
1087 * atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1089 * @v: pointer to atomic_t
1091 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1093 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_relaxed() there.
1095 * Return: The original value of @v.
1097 static __always_inline int
1098 atomic_fetch_xor_relaxed(int i, atomic_t *v)
1100 instrument_atomic_read_write(v, sizeof(*v));
1101 return raw_atomic_fetch_xor_relaxed(i, v);
1105 * atomic_xchg() - atomic exchange with full ordering
1106 * @v: pointer to atomic_t
1107 * @new: int value to assign
1109 * Atomically updates @v to @new with full ordering.
1111 * Unsafe to use in noinstr code; use raw_atomic_xchg() there.
1113 * Return: The original value of @v.
1115 static __always_inline int
1116 atomic_xchg(atomic_t *v, int new)
1119 instrument_atomic_read_write(v, sizeof(*v));
1120 return raw_atomic_xchg(v, new);
1124 * atomic_xchg_acquire() - atomic exchange with acquire ordering
1125 * @v: pointer to atomic_t
1126 * @new: int value to assign
1128 * Atomically updates @v to @new with acquire ordering.
1130 * Unsafe to use in noinstr code; use raw_atomic_xchg_acquire() there.
1132 * Return: The original value of @v.
1134 static __always_inline int
1135 atomic_xchg_acquire(atomic_t *v, int new)
1137 instrument_atomic_read_write(v, sizeof(*v));
1138 return raw_atomic_xchg_acquire(v, new);
1142 * atomic_xchg_release() - atomic exchange with release ordering
1143 * @v: pointer to atomic_t
1144 * @new: int value to assign
1146 * Atomically updates @v to @new with release ordering.
1148 * Unsafe to use in noinstr code; use raw_atomic_xchg_release() there.
1150 * Return: The original value of @v.
1152 static __always_inline int
1153 atomic_xchg_release(atomic_t *v, int new)
1156 instrument_atomic_read_write(v, sizeof(*v));
1157 return raw_atomic_xchg_release(v, new);
1161 * atomic_xchg_relaxed() - atomic exchange with relaxed ordering
1162 * @v: pointer to atomic_t
1163 * @new: int value to assign
1165 * Atomically updates @v to @new with relaxed ordering.
1167 * Unsafe to use in noinstr code; use raw_atomic_xchg_relaxed() there.
1169 * Return: The original value of @v.
1171 static __always_inline int
1172 atomic_xchg_relaxed(atomic_t *v, int new)
1174 instrument_atomic_read_write(v, sizeof(*v));
1175 return raw_atomic_xchg_relaxed(v, new);
1179 * atomic_cmpxchg() - atomic compare and exchange with full ordering
1180 * @v: pointer to atomic_t
1181 * @old: int value to compare with
1182 * @new: int value to assign
1184 * If (@v == @old), atomically updates @v to @new with full ordering.
1185 * Otherwise, @v is not modified and relaxed ordering is provided.
1187 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg() there.
1189 * Return: The original value of @v.
1191 static __always_inline int
1192 atomic_cmpxchg(atomic_t *v, int old, int new)
1195 instrument_atomic_read_write(v, sizeof(*v));
1196 return raw_atomic_cmpxchg(v, old, new);
1200 * atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
1201 * @v: pointer to atomic_t
1202 * @old: int value to compare with
1203 * @new: int value to assign
1205 * If (@v == @old), atomically updates @v to @new with acquire ordering.
1206 * Otherwise, @v is not modified and relaxed ordering is provided.
1208 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_acquire() there.
1210 * Return: The original value of @v.
1212 static __always_inline int
1213 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
1215 instrument_atomic_read_write(v, sizeof(*v));
1216 return raw_atomic_cmpxchg_acquire(v, old, new);
1220 * atomic_cmpxchg_release() - atomic compare and exchange with release ordering
1221 * @v: pointer to atomic_t
1222 * @old: int value to compare with
1223 * @new: int value to assign
1225 * If (@v == @old), atomically updates @v to @new with release ordering.
1226 * Otherwise, @v is not modified and relaxed ordering is provided.
1228 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_release() there.
1230 * Return: The original value of @v.
1232 static __always_inline int
1233 atomic_cmpxchg_release(atomic_t *v, int old, int new)
1236 instrument_atomic_read_write(v, sizeof(*v));
1237 return raw_atomic_cmpxchg_release(v, old, new);
1241 * atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
1242 * @v: pointer to atomic_t
1243 * @old: int value to compare with
1244 * @new: int value to assign
1246 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
1247 * Otherwise, @v is not modified and relaxed ordering is provided.
1249 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_relaxed() there.
1251 * Return: The original value of @v.
1253 static __always_inline int
1254 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
1256 instrument_atomic_read_write(v, sizeof(*v));
1257 return raw_atomic_cmpxchg_relaxed(v, old, new);
1261 * atomic_try_cmpxchg() - atomic compare and exchange with full ordering
1262 * @v: pointer to atomic_t
1263 * @old: pointer to int value to compare with
1264 * @new: int value to assign
1266 * If (@v == @old), atomically updates @v to @new with full ordering.
1267 * Otherwise, @v is not modified, @old is updated to the current value of @v,
1268 * and relaxed ordering is provided.
1270 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there.
1272 * Return: @true if the exchange occured, @false otherwise.
1274 static __always_inline bool
1275 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1278 instrument_atomic_read_write(v, sizeof(*v));
1279 instrument_atomic_read_write(old, sizeof(*old));
1280 return raw_atomic_try_cmpxchg(v, old, new);
1284 * atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
1285 * @v: pointer to atomic_t
1286 * @old: pointer to int value to compare with
1287 * @new: int value to assign
1289 * If (@v == @old), atomically updates @v to @new with acquire ordering.
1290 * Otherwise, @v is not modified, @old is updated to the current value of @v,
1291 * and relaxed ordering is provided.
1293 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there.
1295 * Return: @true if the exchange occured, @false otherwise.
1297 static __always_inline bool
1298 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1300 instrument_atomic_read_write(v, sizeof(*v));
1301 instrument_atomic_read_write(old, sizeof(*old));
1302 return raw_atomic_try_cmpxchg_acquire(v, old, new);
1306 * atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
1307 * @v: pointer to atomic_t
1308 * @old: pointer to int value to compare with
1309 * @new: int value to assign
1311 * If (@v == @old), atomically updates @v to @new with release ordering.
1312 * Otherwise, @v is not modified, @old is updated to the current value of @v,
1313 * and relaxed ordering is provided.
1315 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there.
1317 * Return: @true if the exchange occured, @false otherwise.
1319 static __always_inline bool
1320 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1323 instrument_atomic_read_write(v, sizeof(*v));
1324 instrument_atomic_read_write(old, sizeof(*old));
1325 return raw_atomic_try_cmpxchg_release(v, old, new);
1329 * atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
1330 * @v: pointer to atomic_t
1331 * @old: pointer to int value to compare with
1332 * @new: int value to assign
1334 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
1335 * Otherwise, @v is not modified, @old is updated to the current value of @v,
1336 * and relaxed ordering is provided.
1338 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there.
1340 * Return: @true if the exchange occured, @false otherwise.
1342 static __always_inline bool
1343 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1345 instrument_atomic_read_write(v, sizeof(*v));
1346 instrument_atomic_read_write(old, sizeof(*old));
1347 return raw_atomic_try_cmpxchg_relaxed(v, old, new);
1351 * atomic_sub_and_test() - atomic subtract and test if zero with full ordering
1352 * @i: int value to subtract
1353 * @v: pointer to atomic_t
1355 * Atomically updates @v to (@v - @i) with full ordering.
1357 * Unsafe to use in noinstr code; use raw_atomic_sub_and_test() there.
1359 * Return: @true if the resulting value of @v is zero, @false otherwise.
1361 static __always_inline bool
1362 atomic_sub_and_test(int i, atomic_t *v)
1365 instrument_atomic_read_write(v, sizeof(*v));
1366 return raw_atomic_sub_and_test(i, v);
1370 * atomic_dec_and_test() - atomic decrement and test if zero with full ordering
1371 * @v: pointer to atomic_t
1373 * Atomically updates @v to (@v - 1) with full ordering.
1375 * Unsafe to use in noinstr code; use raw_atomic_dec_and_test() there.
1377 * Return: @true if the resulting value of @v is zero, @false otherwise.
1379 static __always_inline bool
1380 atomic_dec_and_test(atomic_t *v)
1383 instrument_atomic_read_write(v, sizeof(*v));
1384 return raw_atomic_dec_and_test(v);
1388 * atomic_inc_and_test() - atomic increment and test if zero with full ordering
1389 * @v: pointer to atomic_t
1391 * Atomically updates @v to (@v + 1) with full ordering.
1393 * Unsafe to use in noinstr code; use raw_atomic_inc_and_test() there.
1395 * Return: @true if the resulting value of @v is zero, @false otherwise.
1397 static __always_inline bool
1398 atomic_inc_and_test(atomic_t *v)
1401 instrument_atomic_read_write(v, sizeof(*v));
1402 return raw_atomic_inc_and_test(v);
1406 * atomic_add_negative() - atomic add and test if negative with full ordering
1407 * @i: int value to add
1408 * @v: pointer to atomic_t
1410 * Atomically updates @v to (@v + @i) with full ordering.
1412 * Unsafe to use in noinstr code; use raw_atomic_add_negative() there.
1414 * Return: @true if the resulting value of @v is negative, @false otherwise.
1416 static __always_inline bool
1417 atomic_add_negative(int i, atomic_t *v)
1420 instrument_atomic_read_write(v, sizeof(*v));
1421 return raw_atomic_add_negative(i, v);
1425 * atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
1426 * @i: int value to add
1427 * @v: pointer to atomic_t
1429 * Atomically updates @v to (@v + @i) with acquire ordering.
1431 * Unsafe to use in noinstr code; use raw_atomic_add_negative_acquire() there.
1433 * Return: @true if the resulting value of @v is negative, @false otherwise.
1435 static __always_inline bool
1436 atomic_add_negative_acquire(int i, atomic_t *v)
1438 instrument_atomic_read_write(v, sizeof(*v));
1439 return raw_atomic_add_negative_acquire(i, v);
1443 * atomic_add_negative_release() - atomic add and test if negative with release ordering
1444 * @i: int value to add
1445 * @v: pointer to atomic_t
1447 * Atomically updates @v to (@v + @i) with release ordering.
1449 * Unsafe to use in noinstr code; use raw_atomic_add_negative_release() there.
1451 * Return: @true if the resulting value of @v is negative, @false otherwise.
1453 static __always_inline bool
1454 atomic_add_negative_release(int i, atomic_t *v)
1457 instrument_atomic_read_write(v, sizeof(*v));
1458 return raw_atomic_add_negative_release(i, v);
1462 * atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
1463 * @i: int value to add
1464 * @v: pointer to atomic_t
1466 * Atomically updates @v to (@v + @i) with relaxed ordering.
1468 * Unsafe to use in noinstr code; use raw_atomic_add_negative_relaxed() there.
1470 * Return: @true if the resulting value of @v is negative, @false otherwise.
1472 static __always_inline bool
1473 atomic_add_negative_relaxed(int i, atomic_t *v)
1475 instrument_atomic_read_write(v, sizeof(*v));
1476 return raw_atomic_add_negative_relaxed(i, v);
1480 * atomic_fetch_add_unless() - atomic add unless value with full ordering
1481 * @v: pointer to atomic_t
1482 * @a: int value to add
1483 * @u: int value to compare with
1485 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
1486 * Otherwise, @v is not modified and relaxed ordering is provided.
1488 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_unless() there.
1490 * Return: The original value of @v.
1492 static __always_inline int
1493 atomic_fetch_add_unless(atomic_t *v, int a, int u)
1496 instrument_atomic_read_write(v, sizeof(*v));
1497 return raw_atomic_fetch_add_unless(v, a, u);
1501 * atomic_add_unless() - atomic add unless value with full ordering
1502 * @v: pointer to atomic_t
1503 * @a: int value to add
1504 * @u: int value to compare with
1506 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
1507 * Otherwise, @v is not modified and relaxed ordering is provided.
1509 * Unsafe to use in noinstr code; use raw_atomic_add_unless() there.
1511 * Return: @true if @v was updated, @false otherwise.
1513 static __always_inline bool
1514 atomic_add_unless(atomic_t *v, int a, int u)
1517 instrument_atomic_read_write(v, sizeof(*v));
1518 return raw_atomic_add_unless(v, a, u);
1522 * atomic_inc_not_zero() - atomic increment unless zero with full ordering
1523 * @v: pointer to atomic_t
1525 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
1526 * Otherwise, @v is not modified and relaxed ordering is provided.
1528 * Unsafe to use in noinstr code; use raw_atomic_inc_not_zero() there.
1530 * Return: @true if @v was updated, @false otherwise.
1532 static __always_inline bool
1533 atomic_inc_not_zero(atomic_t *v)
1536 instrument_atomic_read_write(v, sizeof(*v));
1537 return raw_atomic_inc_not_zero(v);
1541 * atomic_inc_unless_negative() - atomic increment unless negative with full ordering
1542 * @v: pointer to atomic_t
1544 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
1545 * Otherwise, @v is not modified and relaxed ordering is provided.
1547 * Unsafe to use in noinstr code; use raw_atomic_inc_unless_negative() there.
1549 * Return: @true if @v was updated, @false otherwise.
1551 static __always_inline bool
1552 atomic_inc_unless_negative(atomic_t *v)
1555 instrument_atomic_read_write(v, sizeof(*v));
1556 return raw_atomic_inc_unless_negative(v);
1560 * atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
1561 * @v: pointer to atomic_t
1563 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
1564 * Otherwise, @v is not modified and relaxed ordering is provided.
1566 * Unsafe to use in noinstr code; use raw_atomic_dec_unless_positive() there.
1568 * Return: @true if @v was updated, @false otherwise.
1570 static __always_inline bool
1571 atomic_dec_unless_positive(atomic_t *v)
1574 instrument_atomic_read_write(v, sizeof(*v));
1575 return raw_atomic_dec_unless_positive(v);
1579 * atomic_dec_if_positive() - atomic decrement if positive with full ordering
1580 * @v: pointer to atomic_t
1582 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
1583 * Otherwise, @v is not modified and relaxed ordering is provided.
1585 * Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there.
1587 * Return: The old value of (@v - 1), regardless of whether @v was updated.
1589 static __always_inline int
1590 atomic_dec_if_positive(atomic_t *v)
1593 instrument_atomic_read_write(v, sizeof(*v));
1594 return raw_atomic_dec_if_positive(v);
1598 * atomic64_read() - atomic load with relaxed ordering
1599 * @v: pointer to atomic64_t
1601 * Atomically loads the value of @v with relaxed ordering.
1603 * Unsafe to use in noinstr code; use raw_atomic64_read() there.
1605 * Return: The value loaded from @v.
1607 static __always_inline s64
1608 atomic64_read(const atomic64_t *v)
1610 instrument_atomic_read(v, sizeof(*v));
1611 return raw_atomic64_read(v);
1615 * atomic64_read_acquire() - atomic load with acquire ordering
1616 * @v: pointer to atomic64_t
1618 * Atomically loads the value of @v with acquire ordering.
1620 * Unsafe to use in noinstr code; use raw_atomic64_read_acquire() there.
1622 * Return: The value loaded from @v.
1624 static __always_inline s64
1625 atomic64_read_acquire(const atomic64_t *v)
1627 instrument_atomic_read(v, sizeof(*v));
1628 return raw_atomic64_read_acquire(v);
1632 * atomic64_set() - atomic set with relaxed ordering
1633 * @v: pointer to atomic64_t
1634 * @i: s64 value to assign
1636 * Atomically sets @v to @i with relaxed ordering.
1638 * Unsafe to use in noinstr code; use raw_atomic64_set() there.
1642 static __always_inline void
1643 atomic64_set(atomic64_t *v, s64 i)
1645 instrument_atomic_write(v, sizeof(*v));
1646 raw_atomic64_set(v, i);
1650 * atomic64_set_release() - atomic set with release ordering
1651 * @v: pointer to atomic64_t
1652 * @i: s64 value to assign
1654 * Atomically sets @v to @i with release ordering.
1656 * Unsafe to use in noinstr code; use raw_atomic64_set_release() there.
1660 static __always_inline void
1661 atomic64_set_release(atomic64_t *v, s64 i)
1664 instrument_atomic_write(v, sizeof(*v));
1665 raw_atomic64_set_release(v, i);
1669 * atomic64_add() - atomic add with relaxed ordering
1670 * @i: s64 value to add
1671 * @v: pointer to atomic64_t
1673 * Atomically updates @v to (@v + @i) with relaxed ordering.
1675 * Unsafe to use in noinstr code; use raw_atomic64_add() there.
1679 static __always_inline void
1680 atomic64_add(s64 i, atomic64_t *v)
1682 instrument_atomic_read_write(v, sizeof(*v));
1683 raw_atomic64_add(i, v);
1687 * atomic64_add_return() - atomic add with full ordering
1688 * @i: s64 value to add
1689 * @v: pointer to atomic64_t
1691 * Atomically updates @v to (@v + @i) with full ordering.
1693 * Unsafe to use in noinstr code; use raw_atomic64_add_return() there.
1695 * Return: The updated value of @v.
1697 static __always_inline s64
1698 atomic64_add_return(s64 i, atomic64_t *v)
1701 instrument_atomic_read_write(v, sizeof(*v));
1702 return raw_atomic64_add_return(i, v);
1706 * atomic64_add_return_acquire() - atomic add with acquire ordering
1707 * @i: s64 value to add
1708 * @v: pointer to atomic64_t
1710 * Atomically updates @v to (@v + @i) with acquire ordering.
1712 * Unsafe to use in noinstr code; use raw_atomic64_add_return_acquire() there.
1714 * Return: The updated value of @v.
1716 static __always_inline s64
1717 atomic64_add_return_acquire(s64 i, atomic64_t *v)
1719 instrument_atomic_read_write(v, sizeof(*v));
1720 return raw_atomic64_add_return_acquire(i, v);
1724 * atomic64_add_return_release() - atomic add with release ordering
1725 * @i: s64 value to add
1726 * @v: pointer to atomic64_t
1728 * Atomically updates @v to (@v + @i) with release ordering.
1730 * Unsafe to use in noinstr code; use raw_atomic64_add_return_release() there.
1732 * Return: The updated value of @v.
1734 static __always_inline s64
1735 atomic64_add_return_release(s64 i, atomic64_t *v)
1738 instrument_atomic_read_write(v, sizeof(*v));
1739 return raw_atomic64_add_return_release(i, v);
1743 * atomic64_add_return_relaxed() - atomic add with relaxed ordering
1744 * @i: s64 value to add
1745 * @v: pointer to atomic64_t
1747 * Atomically updates @v to (@v + @i) with relaxed ordering.
1749 * Unsafe to use in noinstr code; use raw_atomic64_add_return_relaxed() there.
1751 * Return: The updated value of @v.
1753 static __always_inline s64
1754 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
1756 instrument_atomic_read_write(v, sizeof(*v));
1757 return raw_atomic64_add_return_relaxed(i, v);
1761 * atomic64_fetch_add() - atomic add with full ordering
1762 * @i: s64 value to add
1763 * @v: pointer to atomic64_t
1765 * Atomically updates @v to (@v + @i) with full ordering.
1767 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add() there.
1769 * Return: The original value of @v.
1771 static __always_inline s64
1772 atomic64_fetch_add(s64 i, atomic64_t *v)
1775 instrument_atomic_read_write(v, sizeof(*v));
1776 return raw_atomic64_fetch_add(i, v);
1780 * atomic64_fetch_add_acquire() - atomic add with acquire ordering
1781 * @i: s64 value to add
1782 * @v: pointer to atomic64_t
1784 * Atomically updates @v to (@v + @i) with acquire ordering.
1786 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_acquire() there.
1788 * Return: The original value of @v.
1790 static __always_inline s64
1791 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1793 instrument_atomic_read_write(v, sizeof(*v));
1794 return raw_atomic64_fetch_add_acquire(i, v);
1798 * atomic64_fetch_add_release() - atomic add with release ordering
1799 * @i: s64 value to add
1800 * @v: pointer to atomic64_t
1802 * Atomically updates @v to (@v + @i) with release ordering.
1804 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_release() there.
1806 * Return: The original value of @v.
1808 static __always_inline s64
1809 atomic64_fetch_add_release(s64 i, atomic64_t *v)
1812 instrument_atomic_read_write(v, sizeof(*v));
1813 return raw_atomic64_fetch_add_release(i, v);
1817 * atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
1818 * @i: s64 value to add
1819 * @v: pointer to atomic64_t
1821 * Atomically updates @v to (@v + @i) with relaxed ordering.
1823 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_relaxed() there.
1825 * Return: The original value of @v.
1827 static __always_inline s64
1828 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
1830 instrument_atomic_read_write(v, sizeof(*v));
1831 return raw_atomic64_fetch_add_relaxed(i, v);
1835 * atomic64_sub() - atomic subtract with relaxed ordering
1836 * @i: s64 value to subtract
1837 * @v: pointer to atomic64_t
1839 * Atomically updates @v to (@v - @i) with relaxed ordering.
1841 * Unsafe to use in noinstr code; use raw_atomic64_sub() there.
1845 static __always_inline void
1846 atomic64_sub(s64 i, atomic64_t *v)
1848 instrument_atomic_read_write(v, sizeof(*v));
1849 raw_atomic64_sub(i, v);
1853 * atomic64_sub_return() - atomic subtract with full ordering
1854 * @i: s64 value to subtract
1855 * @v: pointer to atomic64_t
1857 * Atomically updates @v to (@v - @i) with full ordering.
1859 * Unsafe to use in noinstr code; use raw_atomic64_sub_return() there.
1861 * Return: The updated value of @v.
1863 static __always_inline s64
1864 atomic64_sub_return(s64 i, atomic64_t *v)
1867 instrument_atomic_read_write(v, sizeof(*v));
1868 return raw_atomic64_sub_return(i, v);
1872 * atomic64_sub_return_acquire() - atomic subtract with acquire ordering
1873 * @i: s64 value to subtract
1874 * @v: pointer to atomic64_t
1876 * Atomically updates @v to (@v - @i) with acquire ordering.
1878 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_acquire() there.
1880 * Return: The updated value of @v.
1882 static __always_inline s64
1883 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1885 instrument_atomic_read_write(v, sizeof(*v));
1886 return raw_atomic64_sub_return_acquire(i, v);
1890 * atomic64_sub_return_release() - atomic subtract with release ordering
1891 * @i: s64 value to subtract
1892 * @v: pointer to atomic64_t
1894 * Atomically updates @v to (@v - @i) with release ordering.
1896 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_release() there.
1898 * Return: The updated value of @v.
1900 static __always_inline s64
1901 atomic64_sub_return_release(s64 i, atomic64_t *v)
1904 instrument_atomic_read_write(v, sizeof(*v));
1905 return raw_atomic64_sub_return_release(i, v);
1909 * atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
1910 * @i: s64 value to subtract
1911 * @v: pointer to atomic64_t
1913 * Atomically updates @v to (@v - @i) with relaxed ordering.
1915 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_relaxed() there.
1917 * Return: The updated value of @v.
1919 static __always_inline s64
1920 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
1922 instrument_atomic_read_write(v, sizeof(*v));
1923 return raw_atomic64_sub_return_relaxed(i, v);
1927 * atomic64_fetch_sub() - atomic subtract with full ordering
1928 * @i: s64 value to subtract
1929 * @v: pointer to atomic64_t
1931 * Atomically updates @v to (@v - @i) with full ordering.
1933 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub() there.
1935 * Return: The original value of @v.
1937 static __always_inline s64
1938 atomic64_fetch_sub(s64 i, atomic64_t *v)
1941 instrument_atomic_read_write(v, sizeof(*v));
1942 return raw_atomic64_fetch_sub(i, v);
1946 * atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
1947 * @i: s64 value to subtract
1948 * @v: pointer to atomic64_t
1950 * Atomically updates @v to (@v - @i) with acquire ordering.
1952 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_acquire() there.
1954 * Return: The original value of @v.
1956 static __always_inline s64
1957 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1959 instrument_atomic_read_write(v, sizeof(*v));
1960 return raw_atomic64_fetch_sub_acquire(i, v);
1964 * atomic64_fetch_sub_release() - atomic subtract with release ordering
1965 * @i: s64 value to subtract
1966 * @v: pointer to atomic64_t
1968 * Atomically updates @v to (@v - @i) with release ordering.
1970 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_release() there.
1972 * Return: The original value of @v.
1974 static __always_inline s64
1975 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1978 instrument_atomic_read_write(v, sizeof(*v));
1979 return raw_atomic64_fetch_sub_release(i, v);
1983 * atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
1984 * @i: s64 value to subtract
1985 * @v: pointer to atomic64_t
1987 * Atomically updates @v to (@v - @i) with relaxed ordering.
1989 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_relaxed() there.
1991 * Return: The original value of @v.
1993 static __always_inline s64
1994 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
1996 instrument_atomic_read_write(v, sizeof(*v));
1997 return raw_atomic64_fetch_sub_relaxed(i, v);
2001 * atomic64_inc() - atomic increment with relaxed ordering
2002 * @v: pointer to atomic64_t
2004 * Atomically updates @v to (@v + 1) with relaxed ordering.
2006 * Unsafe to use in noinstr code; use raw_atomic64_inc() there.
2010 static __always_inline void
2011 atomic64_inc(atomic64_t *v)
2013 instrument_atomic_read_write(v, sizeof(*v));
2014 raw_atomic64_inc(v);
2018 * atomic64_inc_return() - atomic increment with full ordering
2019 * @v: pointer to atomic64_t
2021 * Atomically updates @v to (@v + 1) with full ordering.
2023 * Unsafe to use in noinstr code; use raw_atomic64_inc_return() there.
2025 * Return: The updated value of @v.
2027 static __always_inline s64
2028 atomic64_inc_return(atomic64_t *v)
2031 instrument_atomic_read_write(v, sizeof(*v));
2032 return raw_atomic64_inc_return(v);
2036 * atomic64_inc_return_acquire() - atomic increment with acquire ordering
2037 * @v: pointer to atomic64_t
2039 * Atomically updates @v to (@v + 1) with acquire ordering.
2041 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_acquire() there.
2043 * Return: The updated value of @v.
2045 static __always_inline s64
2046 atomic64_inc_return_acquire(atomic64_t *v)
2048 instrument_atomic_read_write(v, sizeof(*v));
2049 return raw_atomic64_inc_return_acquire(v);
2053 * atomic64_inc_return_release() - atomic increment with release ordering
2054 * @v: pointer to atomic64_t
2056 * Atomically updates @v to (@v + 1) with release ordering.
2058 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_release() there.
2060 * Return: The updated value of @v.
2062 static __always_inline s64
2063 atomic64_inc_return_release(atomic64_t *v)
2066 instrument_atomic_read_write(v, sizeof(*v));
2067 return raw_atomic64_inc_return_release(v);
2071 * atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
2072 * @v: pointer to atomic64_t
2074 * Atomically updates @v to (@v + 1) with relaxed ordering.
2076 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_relaxed() there.
2078 * Return: The updated value of @v.
2080 static __always_inline s64
2081 atomic64_inc_return_relaxed(atomic64_t *v)
2083 instrument_atomic_read_write(v, sizeof(*v));
2084 return raw_atomic64_inc_return_relaxed(v);
2088 * atomic64_fetch_inc() - atomic increment with full ordering
2089 * @v: pointer to atomic64_t
2091 * Atomically updates @v to (@v + 1) with full ordering.
2093 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc() there.
2095 * Return: The original value of @v.
2097 static __always_inline s64
2098 atomic64_fetch_inc(atomic64_t *v)
2101 instrument_atomic_read_write(v, sizeof(*v));
2102 return raw_atomic64_fetch_inc(v);
2106 * atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
2107 * @v: pointer to atomic64_t
2109 * Atomically updates @v to (@v + 1) with acquire ordering.
2111 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_acquire() there.
2113 * Return: The original value of @v.
2115 static __always_inline s64
2116 atomic64_fetch_inc_acquire(atomic64_t *v)
2118 instrument_atomic_read_write(v, sizeof(*v));
2119 return raw_atomic64_fetch_inc_acquire(v);
2123 * atomic64_fetch_inc_release() - atomic increment with release ordering
2124 * @v: pointer to atomic64_t
2126 * Atomically updates @v to (@v + 1) with release ordering.
2128 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_release() there.
2130 * Return: The original value of @v.
2132 static __always_inline s64
2133 atomic64_fetch_inc_release(atomic64_t *v)
2136 instrument_atomic_read_write(v, sizeof(*v));
2137 return raw_atomic64_fetch_inc_release(v);
2141 * atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
2142 * @v: pointer to atomic64_t
2144 * Atomically updates @v to (@v + 1) with relaxed ordering.
2146 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_relaxed() there.
2148 * Return: The original value of @v.
2150 static __always_inline s64
2151 atomic64_fetch_inc_relaxed(atomic64_t *v)
2153 instrument_atomic_read_write(v, sizeof(*v));
2154 return raw_atomic64_fetch_inc_relaxed(v);
2158 * atomic64_dec() - atomic decrement with relaxed ordering
2159 * @v: pointer to atomic64_t
2161 * Atomically updates @v to (@v - 1) with relaxed ordering.
2163 * Unsafe to use in noinstr code; use raw_atomic64_dec() there.
2167 static __always_inline void
2168 atomic64_dec(atomic64_t *v)
2170 instrument_atomic_read_write(v, sizeof(*v));
2171 raw_atomic64_dec(v);
2175 * atomic64_dec_return() - atomic decrement with full ordering
2176 * @v: pointer to atomic64_t
2178 * Atomically updates @v to (@v - 1) with full ordering.
2180 * Unsafe to use in noinstr code; use raw_atomic64_dec_return() there.
2182 * Return: The updated value of @v.
2184 static __always_inline s64
2185 atomic64_dec_return(atomic64_t *v)
2188 instrument_atomic_read_write(v, sizeof(*v));
2189 return raw_atomic64_dec_return(v);
2193 * atomic64_dec_return_acquire() - atomic decrement with acquire ordering
2194 * @v: pointer to atomic64_t
2196 * Atomically updates @v to (@v - 1) with acquire ordering.
2198 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_acquire() there.
2200 * Return: The updated value of @v.
2202 static __always_inline s64
2203 atomic64_dec_return_acquire(atomic64_t *v)
2205 instrument_atomic_read_write(v, sizeof(*v));
2206 return raw_atomic64_dec_return_acquire(v);
2210 * atomic64_dec_return_release() - atomic decrement with release ordering
2211 * @v: pointer to atomic64_t
2213 * Atomically updates @v to (@v - 1) with release ordering.
2215 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_release() there.
2217 * Return: The updated value of @v.
2219 static __always_inline s64
2220 atomic64_dec_return_release(atomic64_t *v)
2223 instrument_atomic_read_write(v, sizeof(*v));
2224 return raw_atomic64_dec_return_release(v);
2228 * atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
2229 * @v: pointer to atomic64_t
2231 * Atomically updates @v to (@v - 1) with relaxed ordering.
2233 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_relaxed() there.
2235 * Return: The updated value of @v.
2237 static __always_inline s64
2238 atomic64_dec_return_relaxed(atomic64_t *v)
2240 instrument_atomic_read_write(v, sizeof(*v));
2241 return raw_atomic64_dec_return_relaxed(v);
2245 * atomic64_fetch_dec() - atomic decrement with full ordering
2246 * @v: pointer to atomic64_t
2248 * Atomically updates @v to (@v - 1) with full ordering.
2250 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec() there.
2252 * Return: The original value of @v.
2254 static __always_inline s64
2255 atomic64_fetch_dec(atomic64_t *v)
2258 instrument_atomic_read_write(v, sizeof(*v));
2259 return raw_atomic64_fetch_dec(v);
2263 * atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
2264 * @v: pointer to atomic64_t
2266 * Atomically updates @v to (@v - 1) with acquire ordering.
2268 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_acquire() there.
2270 * Return: The original value of @v.
2272 static __always_inline s64
2273 atomic64_fetch_dec_acquire(atomic64_t *v)
2275 instrument_atomic_read_write(v, sizeof(*v));
2276 return raw_atomic64_fetch_dec_acquire(v);
2280 * atomic64_fetch_dec_release() - atomic decrement with release ordering
2281 * @v: pointer to atomic64_t
2283 * Atomically updates @v to (@v - 1) with release ordering.
2285 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_release() there.
2287 * Return: The original value of @v.
2289 static __always_inline s64
2290 atomic64_fetch_dec_release(atomic64_t *v)
2293 instrument_atomic_read_write(v, sizeof(*v));
2294 return raw_atomic64_fetch_dec_release(v);
2298 * atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
2299 * @v: pointer to atomic64_t
2301 * Atomically updates @v to (@v - 1) with relaxed ordering.
2303 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_relaxed() there.
2305 * Return: The original value of @v.
2307 static __always_inline s64
2308 atomic64_fetch_dec_relaxed(atomic64_t *v)
2310 instrument_atomic_read_write(v, sizeof(*v));
2311 return raw_atomic64_fetch_dec_relaxed(v);
2315 * atomic64_and() - atomic bitwise AND with relaxed ordering
2317 * @v: pointer to atomic64_t
2319 * Atomically updates @v to (@v & @i) with relaxed ordering.
2321 * Unsafe to use in noinstr code; use raw_atomic64_and() there.
2325 static __always_inline void
2326 atomic64_and(s64 i, atomic64_t *v)
2328 instrument_atomic_read_write(v, sizeof(*v));
2329 raw_atomic64_and(i, v);
2333 * atomic64_fetch_and() - atomic bitwise AND with full ordering
2335 * @v: pointer to atomic64_t
2337 * Atomically updates @v to (@v & @i) with full ordering.
2339 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and() there.
2341 * Return: The original value of @v.
2343 static __always_inline s64
2344 atomic64_fetch_and(s64 i, atomic64_t *v)
2347 instrument_atomic_read_write(v, sizeof(*v));
2348 return raw_atomic64_fetch_and(i, v);
2352 * atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
2354 * @v: pointer to atomic64_t
2356 * Atomically updates @v to (@v & @i) with acquire ordering.
2358 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_acquire() there.
2360 * Return: The original value of @v.
2362 static __always_inline s64
2363 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
2365 instrument_atomic_read_write(v, sizeof(*v));
2366 return raw_atomic64_fetch_and_acquire(i, v);
2370 * atomic64_fetch_and_release() - atomic bitwise AND with release ordering
2372 * @v: pointer to atomic64_t
2374 * Atomically updates @v to (@v & @i) with release ordering.
2376 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_release() there.
2378 * Return: The original value of @v.
2380 static __always_inline s64
2381 atomic64_fetch_and_release(s64 i, atomic64_t *v)
2384 instrument_atomic_read_write(v, sizeof(*v));
2385 return raw_atomic64_fetch_and_release(i, v);
2389 * atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
2391 * @v: pointer to atomic64_t
2393 * Atomically updates @v to (@v & @i) with relaxed ordering.
2395 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_relaxed() there.
2397 * Return: The original value of @v.
2399 static __always_inline s64
2400 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
2402 instrument_atomic_read_write(v, sizeof(*v));
2403 return raw_atomic64_fetch_and_relaxed(i, v);
2407 * atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
2409 * @v: pointer to atomic64_t
2411 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
2413 * Unsafe to use in noinstr code; use raw_atomic64_andnot() there.
2417 static __always_inline void
2418 atomic64_andnot(s64 i, atomic64_t *v)
2420 instrument_atomic_read_write(v, sizeof(*v));
2421 raw_atomic64_andnot(i, v);
2425 * atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
2427 * @v: pointer to atomic64_t
2429 * Atomically updates @v to (@v & ~@i) with full ordering.
2431 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot() there.
2433 * Return: The original value of @v.
2435 static __always_inline s64
2436 atomic64_fetch_andnot(s64 i, atomic64_t *v)
2439 instrument_atomic_read_write(v, sizeof(*v));
2440 return raw_atomic64_fetch_andnot(i, v);
2444 * atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
2446 * @v: pointer to atomic64_t
2448 * Atomically updates @v to (@v & ~@i) with acquire ordering.
2450 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_acquire() there.
2452 * Return: The original value of @v.
2454 static __always_inline s64
2455 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
2457 instrument_atomic_read_write(v, sizeof(*v));
2458 return raw_atomic64_fetch_andnot_acquire(i, v);
2462 * atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
2464 * @v: pointer to atomic64_t
2466 * Atomically updates @v to (@v & ~@i) with release ordering.
2468 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_release() there.
2470 * Return: The original value of @v.
2472 static __always_inline s64
2473 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
2476 instrument_atomic_read_write(v, sizeof(*v));
2477 return raw_atomic64_fetch_andnot_release(i, v);
2481 * atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
2483 * @v: pointer to atomic64_t
2485 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
2487 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_relaxed() there.
2489 * Return: The original value of @v.
2491 static __always_inline s64
2492 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
2494 instrument_atomic_read_write(v, sizeof(*v));
2495 return raw_atomic64_fetch_andnot_relaxed(i, v);
2499 * atomic64_or() - atomic bitwise OR with relaxed ordering
2501 * @v: pointer to atomic64_t
2503 * Atomically updates @v to (@v | @i) with relaxed ordering.
2505 * Unsafe to use in noinstr code; use raw_atomic64_or() there.
2509 static __always_inline void
2510 atomic64_or(s64 i, atomic64_t *v)
2512 instrument_atomic_read_write(v, sizeof(*v));
2513 raw_atomic64_or(i, v);
2517 * atomic64_fetch_or() - atomic bitwise OR with full ordering
2519 * @v: pointer to atomic64_t
2521 * Atomically updates @v to (@v | @i) with full ordering.
2523 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or() there.
2525 * Return: The original value of @v.
2527 static __always_inline s64
2528 atomic64_fetch_or(s64 i, atomic64_t *v)
2531 instrument_atomic_read_write(v, sizeof(*v));
2532 return raw_atomic64_fetch_or(i, v);
2536 * atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
2538 * @v: pointer to atomic64_t
2540 * Atomically updates @v to (@v | @i) with acquire ordering.
2542 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_acquire() there.
2544 * Return: The original value of @v.
2546 static __always_inline s64
2547 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
2549 instrument_atomic_read_write(v, sizeof(*v));
2550 return raw_atomic64_fetch_or_acquire(i, v);
2554 * atomic64_fetch_or_release() - atomic bitwise OR with release ordering
2556 * @v: pointer to atomic64_t
2558 * Atomically updates @v to (@v | @i) with release ordering.
2560 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_release() there.
2562 * Return: The original value of @v.
2564 static __always_inline s64
2565 atomic64_fetch_or_release(s64 i, atomic64_t *v)
2568 instrument_atomic_read_write(v, sizeof(*v));
2569 return raw_atomic64_fetch_or_release(i, v);
2573 * atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
2575 * @v: pointer to atomic64_t
2577 * Atomically updates @v to (@v | @i) with relaxed ordering.
2579 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_relaxed() there.
2581 * Return: The original value of @v.
2583 static __always_inline s64
2584 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
2586 instrument_atomic_read_write(v, sizeof(*v));
2587 return raw_atomic64_fetch_or_relaxed(i, v);
2591 * atomic64_xor() - atomic bitwise XOR with relaxed ordering
2593 * @v: pointer to atomic64_t
2595 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
2597 * Unsafe to use in noinstr code; use raw_atomic64_xor() there.
2601 static __always_inline void
2602 atomic64_xor(s64 i, atomic64_t *v)
2604 instrument_atomic_read_write(v, sizeof(*v));
2605 raw_atomic64_xor(i, v);
2609 * atomic64_fetch_xor() - atomic bitwise XOR with full ordering
2611 * @v: pointer to atomic64_t
2613 * Atomically updates @v to (@v ^ @i) with full ordering.
2615 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor() there.
2617 * Return: The original value of @v.
2619 static __always_inline s64
2620 atomic64_fetch_xor(s64 i, atomic64_t *v)
2623 instrument_atomic_read_write(v, sizeof(*v));
2624 return raw_atomic64_fetch_xor(i, v);
2628 * atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
2630 * @v: pointer to atomic64_t
2632 * Atomically updates @v to (@v ^ @i) with acquire ordering.
2634 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_acquire() there.
2636 * Return: The original value of @v.
2638 static __always_inline s64
2639 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
2641 instrument_atomic_read_write(v, sizeof(*v));
2642 return raw_atomic64_fetch_xor_acquire(i, v);
2646 * atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
2648 * @v: pointer to atomic64_t
2650 * Atomically updates @v to (@v ^ @i) with release ordering.
2652 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_release() there.
2654 * Return: The original value of @v.
2656 static __always_inline s64
2657 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
2660 instrument_atomic_read_write(v, sizeof(*v));
2661 return raw_atomic64_fetch_xor_release(i, v);
2665 * atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
2667 * @v: pointer to atomic64_t
2669 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
2671 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_relaxed() there.
2673 * Return: The original value of @v.
2675 static __always_inline s64
2676 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
2678 instrument_atomic_read_write(v, sizeof(*v));
2679 return raw_atomic64_fetch_xor_relaxed(i, v);
2683 * atomic64_xchg() - atomic exchange with full ordering
2684 * @v: pointer to atomic64_t
2685 * @new: s64 value to assign
2687 * Atomically updates @v to @new with full ordering.
2689 * Unsafe to use in noinstr code; use raw_atomic64_xchg() there.
2691 * Return: The original value of @v.
2693 static __always_inline s64
2694 atomic64_xchg(atomic64_t *v, s64 new)
2697 instrument_atomic_read_write(v, sizeof(*v));
2698 return raw_atomic64_xchg(v, new);
2702 * atomic64_xchg_acquire() - atomic exchange with acquire ordering
2703 * @v: pointer to atomic64_t
2704 * @new: s64 value to assign
2706 * Atomically updates @v to @new with acquire ordering.
2708 * Unsafe to use in noinstr code; use raw_atomic64_xchg_acquire() there.
2710 * Return: The original value of @v.
2712 static __always_inline s64
2713 atomic64_xchg_acquire(atomic64_t *v, s64 new)
2715 instrument_atomic_read_write(v, sizeof(*v));
2716 return raw_atomic64_xchg_acquire(v, new);
2720 * atomic64_xchg_release() - atomic exchange with release ordering
2721 * @v: pointer to atomic64_t
2722 * @new: s64 value to assign
2724 * Atomically updates @v to @new with release ordering.
2726 * Unsafe to use in noinstr code; use raw_atomic64_xchg_release() there.
2728 * Return: The original value of @v.
2730 static __always_inline s64
2731 atomic64_xchg_release(atomic64_t *v, s64 new)
2734 instrument_atomic_read_write(v, sizeof(*v));
2735 return raw_atomic64_xchg_release(v, new);
2739 * atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
2740 * @v: pointer to atomic64_t
2741 * @new: s64 value to assign
2743 * Atomically updates @v to @new with relaxed ordering.
2745 * Unsafe to use in noinstr code; use raw_atomic64_xchg_relaxed() there.
2747 * Return: The original value of @v.
2749 static __always_inline s64
2750 atomic64_xchg_relaxed(atomic64_t *v, s64 new)
2752 instrument_atomic_read_write(v, sizeof(*v));
2753 return raw_atomic64_xchg_relaxed(v, new);
2757 * atomic64_cmpxchg() - atomic compare and exchange with full ordering
2758 * @v: pointer to atomic64_t
2759 * @old: s64 value to compare with
2760 * @new: s64 value to assign
2762 * If (@v == @old), atomically updates @v to @new with full ordering.
2763 * Otherwise, @v is not modified and relaxed ordering is provided.
2765 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg() there.
2767 * Return: The original value of @v.
2769 static __always_inline s64
2770 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2773 instrument_atomic_read_write(v, sizeof(*v));
2774 return raw_atomic64_cmpxchg(v, old, new);
2778 * atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2779 * @v: pointer to atomic64_t
2780 * @old: s64 value to compare with
2781 * @new: s64 value to assign
2783 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2784 * Otherwise, @v is not modified and relaxed ordering is provided.
2786 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_acquire() there.
2788 * Return: The original value of @v.
2790 static __always_inline s64
2791 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
2793 instrument_atomic_read_write(v, sizeof(*v));
2794 return raw_atomic64_cmpxchg_acquire(v, old, new);
2798 * atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
2799 * @v: pointer to atomic64_t
2800 * @old: s64 value to compare with
2801 * @new: s64 value to assign
2803 * If (@v == @old), atomically updates @v to @new with release ordering.
2804 * Otherwise, @v is not modified and relaxed ordering is provided.
2806 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_release() there.
2808 * Return: The original value of @v.
2810 static __always_inline s64
2811 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
2814 instrument_atomic_read_write(v, sizeof(*v));
2815 return raw_atomic64_cmpxchg_release(v, old, new);
2819 * atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2820 * @v: pointer to atomic64_t
2821 * @old: s64 value to compare with
2822 * @new: s64 value to assign
2824 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2825 * Otherwise, @v is not modified and relaxed ordering is provided.
2827 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_relaxed() there.
2829 * Return: The original value of @v.
2831 static __always_inline s64
2832 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
2834 instrument_atomic_read_write(v, sizeof(*v));
2835 return raw_atomic64_cmpxchg_relaxed(v, old, new);
2839 * atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
2840 * @v: pointer to atomic64_t
2841 * @old: pointer to s64 value to compare with
2842 * @new: s64 value to assign
2844 * If (@v == @old), atomically updates @v to @new with full ordering.
2845 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2846 * and relaxed ordering is provided.
2848 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there.
2850 * Return: @true if the exchange occured, @false otherwise.
2852 static __always_inline bool
2853 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2856 instrument_atomic_read_write(v, sizeof(*v));
2857 instrument_atomic_read_write(old, sizeof(*old));
2858 return raw_atomic64_try_cmpxchg(v, old, new);
2862 * atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2863 * @v: pointer to atomic64_t
2864 * @old: pointer to s64 value to compare with
2865 * @new: s64 value to assign
2867 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2868 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2869 * and relaxed ordering is provided.
2871 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there.
2873 * Return: @true if the exchange occured, @false otherwise.
2875 static __always_inline bool
2876 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2878 instrument_atomic_read_write(v, sizeof(*v));
2879 instrument_atomic_read_write(old, sizeof(*old));
2880 return raw_atomic64_try_cmpxchg_acquire(v, old, new);
2884 * atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
2885 * @v: pointer to atomic64_t
2886 * @old: pointer to s64 value to compare with
2887 * @new: s64 value to assign
2889 * If (@v == @old), atomically updates @v to @new with release ordering.
2890 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2891 * and relaxed ordering is provided.
2893 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there.
2895 * Return: @true if the exchange occured, @false otherwise.
2897 static __always_inline bool
2898 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2901 instrument_atomic_read_write(v, sizeof(*v));
2902 instrument_atomic_read_write(old, sizeof(*old));
2903 return raw_atomic64_try_cmpxchg_release(v, old, new);
2907 * atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2908 * @v: pointer to atomic64_t
2909 * @old: pointer to s64 value to compare with
2910 * @new: s64 value to assign
2912 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2913 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2914 * and relaxed ordering is provided.
2916 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there.
2918 * Return: @true if the exchange occured, @false otherwise.
2920 static __always_inline bool
2921 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2923 instrument_atomic_read_write(v, sizeof(*v));
2924 instrument_atomic_read_write(old, sizeof(*old));
2925 return raw_atomic64_try_cmpxchg_relaxed(v, old, new);
2929 * atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
2930 * @i: s64 value to subtract
2931 * @v: pointer to atomic64_t
2933 * Atomically updates @v to (@v - @i) with full ordering.
2935 * Unsafe to use in noinstr code; use raw_atomic64_sub_and_test() there.
2937 * Return: @true if the resulting value of @v is zero, @false otherwise.
2939 static __always_inline bool
2940 atomic64_sub_and_test(s64 i, atomic64_t *v)
2943 instrument_atomic_read_write(v, sizeof(*v));
2944 return raw_atomic64_sub_and_test(i, v);
2948 * atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
2949 * @v: pointer to atomic64_t
2951 * Atomically updates @v to (@v - 1) with full ordering.
2953 * Unsafe to use in noinstr code; use raw_atomic64_dec_and_test() there.
2955 * Return: @true if the resulting value of @v is zero, @false otherwise.
2957 static __always_inline bool
2958 atomic64_dec_and_test(atomic64_t *v)
2961 instrument_atomic_read_write(v, sizeof(*v));
2962 return raw_atomic64_dec_and_test(v);
2966 * atomic64_inc_and_test() - atomic increment and test if zero with full ordering
2967 * @v: pointer to atomic64_t
2969 * Atomically updates @v to (@v + 1) with full ordering.
2971 * Unsafe to use in noinstr code; use raw_atomic64_inc_and_test() there.
2973 * Return: @true if the resulting value of @v is zero, @false otherwise.
2975 static __always_inline bool
2976 atomic64_inc_and_test(atomic64_t *v)
2979 instrument_atomic_read_write(v, sizeof(*v));
2980 return raw_atomic64_inc_and_test(v);
2984 * atomic64_add_negative() - atomic add and test if negative with full ordering
2985 * @i: s64 value to add
2986 * @v: pointer to atomic64_t
2988 * Atomically updates @v to (@v + @i) with full ordering.
2990 * Unsafe to use in noinstr code; use raw_atomic64_add_negative() there.
2992 * Return: @true if the resulting value of @v is negative, @false otherwise.
2994 static __always_inline bool
2995 atomic64_add_negative(s64 i, atomic64_t *v)
2998 instrument_atomic_read_write(v, sizeof(*v));
2999 return raw_atomic64_add_negative(i, v);
3003 * atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
3004 * @i: s64 value to add
3005 * @v: pointer to atomic64_t
3007 * Atomically updates @v to (@v + @i) with acquire ordering.
3009 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_acquire() there.
3011 * Return: @true if the resulting value of @v is negative, @false otherwise.
3013 static __always_inline bool
3014 atomic64_add_negative_acquire(s64 i, atomic64_t *v)
3016 instrument_atomic_read_write(v, sizeof(*v));
3017 return raw_atomic64_add_negative_acquire(i, v);
3021 * atomic64_add_negative_release() - atomic add and test if negative with release ordering
3022 * @i: s64 value to add
3023 * @v: pointer to atomic64_t
3025 * Atomically updates @v to (@v + @i) with release ordering.
3027 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_release() there.
3029 * Return: @true if the resulting value of @v is negative, @false otherwise.
3031 static __always_inline bool
3032 atomic64_add_negative_release(s64 i, atomic64_t *v)
3035 instrument_atomic_read_write(v, sizeof(*v));
3036 return raw_atomic64_add_negative_release(i, v);
3040 * atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
3041 * @i: s64 value to add
3042 * @v: pointer to atomic64_t
3044 * Atomically updates @v to (@v + @i) with relaxed ordering.
3046 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_relaxed() there.
3048 * Return: @true if the resulting value of @v is negative, @false otherwise.
3050 static __always_inline bool
3051 atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
3053 instrument_atomic_read_write(v, sizeof(*v));
3054 return raw_atomic64_add_negative_relaxed(i, v);
3058 * atomic64_fetch_add_unless() - atomic add unless value with full ordering
3059 * @v: pointer to atomic64_t
3060 * @a: s64 value to add
3061 * @u: s64 value to compare with
3063 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
3064 * Otherwise, @v is not modified and relaxed ordering is provided.
3066 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_unless() there.
3068 * Return: The original value of @v.
3070 static __always_inline s64
3071 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
3074 instrument_atomic_read_write(v, sizeof(*v));
3075 return raw_atomic64_fetch_add_unless(v, a, u);
3079 * atomic64_add_unless() - atomic add unless value with full ordering
3080 * @v: pointer to atomic64_t
3081 * @a: s64 value to add
3082 * @u: s64 value to compare with
3084 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
3085 * Otherwise, @v is not modified and relaxed ordering is provided.
3087 * Unsafe to use in noinstr code; use raw_atomic64_add_unless() there.
3089 * Return: @true if @v was updated, @false otherwise.
3091 static __always_inline bool
3092 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
3095 instrument_atomic_read_write(v, sizeof(*v));
3096 return raw_atomic64_add_unless(v, a, u);
3100 * atomic64_inc_not_zero() - atomic increment unless zero with full ordering
3101 * @v: pointer to atomic64_t
3103 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
3104 * Otherwise, @v is not modified and relaxed ordering is provided.
3106 * Unsafe to use in noinstr code; use raw_atomic64_inc_not_zero() there.
3108 * Return: @true if @v was updated, @false otherwise.
3110 static __always_inline bool
3111 atomic64_inc_not_zero(atomic64_t *v)
3114 instrument_atomic_read_write(v, sizeof(*v));
3115 return raw_atomic64_inc_not_zero(v);
3119 * atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
3120 * @v: pointer to atomic64_t
3122 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
3123 * Otherwise, @v is not modified and relaxed ordering is provided.
3125 * Unsafe to use in noinstr code; use raw_atomic64_inc_unless_negative() there.
3127 * Return: @true if @v was updated, @false otherwise.
3129 static __always_inline bool
3130 atomic64_inc_unless_negative(atomic64_t *v)
3133 instrument_atomic_read_write(v, sizeof(*v));
3134 return raw_atomic64_inc_unless_negative(v);
3138 * atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
3139 * @v: pointer to atomic64_t
3141 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
3142 * Otherwise, @v is not modified and relaxed ordering is provided.
3144 * Unsafe to use in noinstr code; use raw_atomic64_dec_unless_positive() there.
3146 * Return: @true if @v was updated, @false otherwise.
3148 static __always_inline bool
3149 atomic64_dec_unless_positive(atomic64_t *v)
3152 instrument_atomic_read_write(v, sizeof(*v));
3153 return raw_atomic64_dec_unless_positive(v);
3157 * atomic64_dec_if_positive() - atomic decrement if positive with full ordering
3158 * @v: pointer to atomic64_t
3160 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
3161 * Otherwise, @v is not modified and relaxed ordering is provided.
3163 * Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there.
3165 * Return: The old value of (@v - 1), regardless of whether @v was updated.
3167 static __always_inline s64
3168 atomic64_dec_if_positive(atomic64_t *v)
3171 instrument_atomic_read_write(v, sizeof(*v));
3172 return raw_atomic64_dec_if_positive(v);
3176 * atomic_long_read() - atomic load with relaxed ordering
3177 * @v: pointer to atomic_long_t
3179 * Atomically loads the value of @v with relaxed ordering.
3181 * Unsafe to use in noinstr code; use raw_atomic_long_read() there.
3183 * Return: The value loaded from @v.
3185 static __always_inline long
3186 atomic_long_read(const atomic_long_t *v)
3188 instrument_atomic_read(v, sizeof(*v));
3189 return raw_atomic_long_read(v);
3193 * atomic_long_read_acquire() - atomic load with acquire ordering
3194 * @v: pointer to atomic_long_t
3196 * Atomically loads the value of @v with acquire ordering.
3198 * Unsafe to use in noinstr code; use raw_atomic_long_read_acquire() there.
3200 * Return: The value loaded from @v.
3202 static __always_inline long
3203 atomic_long_read_acquire(const atomic_long_t *v)
3205 instrument_atomic_read(v, sizeof(*v));
3206 return raw_atomic_long_read_acquire(v);
3210 * atomic_long_set() - atomic set with relaxed ordering
3211 * @v: pointer to atomic_long_t
3212 * @i: long value to assign
3214 * Atomically sets @v to @i with relaxed ordering.
3216 * Unsafe to use in noinstr code; use raw_atomic_long_set() there.
3220 static __always_inline void
3221 atomic_long_set(atomic_long_t *v, long i)
3223 instrument_atomic_write(v, sizeof(*v));
3224 raw_atomic_long_set(v, i);
3228 * atomic_long_set_release() - atomic set with release ordering
3229 * @v: pointer to atomic_long_t
3230 * @i: long value to assign
3232 * Atomically sets @v to @i with release ordering.
3234 * Unsafe to use in noinstr code; use raw_atomic_long_set_release() there.
3238 static __always_inline void
3239 atomic_long_set_release(atomic_long_t *v, long i)
3242 instrument_atomic_write(v, sizeof(*v));
3243 raw_atomic_long_set_release(v, i);
3247 * atomic_long_add() - atomic add with relaxed ordering
3248 * @i: long value to add
3249 * @v: pointer to atomic_long_t
3251 * Atomically updates @v to (@v + @i) with relaxed ordering.
3253 * Unsafe to use in noinstr code; use raw_atomic_long_add() there.
3257 static __always_inline void
3258 atomic_long_add(long i, atomic_long_t *v)
3260 instrument_atomic_read_write(v, sizeof(*v));
3261 raw_atomic_long_add(i, v);
3265 * atomic_long_add_return() - atomic add with full ordering
3266 * @i: long value to add
3267 * @v: pointer to atomic_long_t
3269 * Atomically updates @v to (@v + @i) with full ordering.
3271 * Unsafe to use in noinstr code; use raw_atomic_long_add_return() there.
3273 * Return: The updated value of @v.
3275 static __always_inline long
3276 atomic_long_add_return(long i, atomic_long_t *v)
3279 instrument_atomic_read_write(v, sizeof(*v));
3280 return raw_atomic_long_add_return(i, v);
3284 * atomic_long_add_return_acquire() - atomic add with acquire ordering
3285 * @i: long value to add
3286 * @v: pointer to atomic_long_t
3288 * Atomically updates @v to (@v + @i) with acquire ordering.
3290 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_acquire() there.
3292 * Return: The updated value of @v.
3294 static __always_inline long
3295 atomic_long_add_return_acquire(long i, atomic_long_t *v)
3297 instrument_atomic_read_write(v, sizeof(*v));
3298 return raw_atomic_long_add_return_acquire(i, v);
3302 * atomic_long_add_return_release() - atomic add with release ordering
3303 * @i: long value to add
3304 * @v: pointer to atomic_long_t
3306 * Atomically updates @v to (@v + @i) with release ordering.
3308 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_release() there.
3310 * Return: The updated value of @v.
3312 static __always_inline long
3313 atomic_long_add_return_release(long i, atomic_long_t *v)
3316 instrument_atomic_read_write(v, sizeof(*v));
3317 return raw_atomic_long_add_return_release(i, v);
3321 * atomic_long_add_return_relaxed() - atomic add with relaxed ordering
3322 * @i: long value to add
3323 * @v: pointer to atomic_long_t
3325 * Atomically updates @v to (@v + @i) with relaxed ordering.
3327 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_relaxed() there.
3329 * Return: The updated value of @v.
3331 static __always_inline long
3332 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
3334 instrument_atomic_read_write(v, sizeof(*v));
3335 return raw_atomic_long_add_return_relaxed(i, v);
3339 * atomic_long_fetch_add() - atomic add with full ordering
3340 * @i: long value to add
3341 * @v: pointer to atomic_long_t
3343 * Atomically updates @v to (@v + @i) with full ordering.
3345 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add() there.
3347 * Return: The original value of @v.
3349 static __always_inline long
3350 atomic_long_fetch_add(long i, atomic_long_t *v)
3353 instrument_atomic_read_write(v, sizeof(*v));
3354 return raw_atomic_long_fetch_add(i, v);
3358 * atomic_long_fetch_add_acquire() - atomic add with acquire ordering
3359 * @i: long value to add
3360 * @v: pointer to atomic_long_t
3362 * Atomically updates @v to (@v + @i) with acquire ordering.
3364 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_acquire() there.
3366 * Return: The original value of @v.
3368 static __always_inline long
3369 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
3371 instrument_atomic_read_write(v, sizeof(*v));
3372 return raw_atomic_long_fetch_add_acquire(i, v);
3376 * atomic_long_fetch_add_release() - atomic add with release ordering
3377 * @i: long value to add
3378 * @v: pointer to atomic_long_t
3380 * Atomically updates @v to (@v + @i) with release ordering.
3382 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_release() there.
3384 * Return: The original value of @v.
3386 static __always_inline long
3387 atomic_long_fetch_add_release(long i, atomic_long_t *v)
3390 instrument_atomic_read_write(v, sizeof(*v));
3391 return raw_atomic_long_fetch_add_release(i, v);
3395 * atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
3396 * @i: long value to add
3397 * @v: pointer to atomic_long_t
3399 * Atomically updates @v to (@v + @i) with relaxed ordering.
3401 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_relaxed() there.
3403 * Return: The original value of @v.
3405 static __always_inline long
3406 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
3408 instrument_atomic_read_write(v, sizeof(*v));
3409 return raw_atomic_long_fetch_add_relaxed(i, v);
3413 * atomic_long_sub() - atomic subtract with relaxed ordering
3414 * @i: long value to subtract
3415 * @v: pointer to atomic_long_t
3417 * Atomically updates @v to (@v - @i) with relaxed ordering.
3419 * Unsafe to use in noinstr code; use raw_atomic_long_sub() there.
3423 static __always_inline void
3424 atomic_long_sub(long i, atomic_long_t *v)
3426 instrument_atomic_read_write(v, sizeof(*v));
3427 raw_atomic_long_sub(i, v);
3431 * atomic_long_sub_return() - atomic subtract with full ordering
3432 * @i: long value to subtract
3433 * @v: pointer to atomic_long_t
3435 * Atomically updates @v to (@v - @i) with full ordering.
3437 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return() there.
3439 * Return: The updated value of @v.
3441 static __always_inline long
3442 atomic_long_sub_return(long i, atomic_long_t *v)
3445 instrument_atomic_read_write(v, sizeof(*v));
3446 return raw_atomic_long_sub_return(i, v);
3450 * atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
3451 * @i: long value to subtract
3452 * @v: pointer to atomic_long_t
3454 * Atomically updates @v to (@v - @i) with acquire ordering.
3456 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_acquire() there.
3458 * Return: The updated value of @v.
3460 static __always_inline long
3461 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
3463 instrument_atomic_read_write(v, sizeof(*v));
3464 return raw_atomic_long_sub_return_acquire(i, v);
3468 * atomic_long_sub_return_release() - atomic subtract with release ordering
3469 * @i: long value to subtract
3470 * @v: pointer to atomic_long_t
3472 * Atomically updates @v to (@v - @i) with release ordering.
3474 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_release() there.
3476 * Return: The updated value of @v.
3478 static __always_inline long
3479 atomic_long_sub_return_release(long i, atomic_long_t *v)
3482 instrument_atomic_read_write(v, sizeof(*v));
3483 return raw_atomic_long_sub_return_release(i, v);
3487 * atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
3488 * @i: long value to subtract
3489 * @v: pointer to atomic_long_t
3491 * Atomically updates @v to (@v - @i) with relaxed ordering.
3493 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_relaxed() there.
3495 * Return: The updated value of @v.
3497 static __always_inline long
3498 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
3500 instrument_atomic_read_write(v, sizeof(*v));
3501 return raw_atomic_long_sub_return_relaxed(i, v);
3505 * atomic_long_fetch_sub() - atomic subtract with full ordering
3506 * @i: long value to subtract
3507 * @v: pointer to atomic_long_t
3509 * Atomically updates @v to (@v - @i) with full ordering.
3511 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub() there.
3513 * Return: The original value of @v.
3515 static __always_inline long
3516 atomic_long_fetch_sub(long i, atomic_long_t *v)
3519 instrument_atomic_read_write(v, sizeof(*v));
3520 return raw_atomic_long_fetch_sub(i, v);
3524 * atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
3525 * @i: long value to subtract
3526 * @v: pointer to atomic_long_t
3528 * Atomically updates @v to (@v - @i) with acquire ordering.
3530 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_acquire() there.
3532 * Return: The original value of @v.
3534 static __always_inline long
3535 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
3537 instrument_atomic_read_write(v, sizeof(*v));
3538 return raw_atomic_long_fetch_sub_acquire(i, v);
3542 * atomic_long_fetch_sub_release() - atomic subtract with release ordering
3543 * @i: long value to subtract
3544 * @v: pointer to atomic_long_t
3546 * Atomically updates @v to (@v - @i) with release ordering.
3548 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_release() there.
3550 * Return: The original value of @v.
3552 static __always_inline long
3553 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
3556 instrument_atomic_read_write(v, sizeof(*v));
3557 return raw_atomic_long_fetch_sub_release(i, v);
3561 * atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
3562 * @i: long value to subtract
3563 * @v: pointer to atomic_long_t
3565 * Atomically updates @v to (@v - @i) with relaxed ordering.
3567 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_relaxed() there.
3569 * Return: The original value of @v.
3571 static __always_inline long
3572 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
3574 instrument_atomic_read_write(v, sizeof(*v));
3575 return raw_atomic_long_fetch_sub_relaxed(i, v);
3579 * atomic_long_inc() - atomic increment with relaxed ordering
3580 * @v: pointer to atomic_long_t
3582 * Atomically updates @v to (@v + 1) with relaxed ordering.
3584 * Unsafe to use in noinstr code; use raw_atomic_long_inc() there.
3588 static __always_inline void
3589 atomic_long_inc(atomic_long_t *v)
3591 instrument_atomic_read_write(v, sizeof(*v));
3592 raw_atomic_long_inc(v);
3596 * atomic_long_inc_return() - atomic increment with full ordering
3597 * @v: pointer to atomic_long_t
3599 * Atomically updates @v to (@v + 1) with full ordering.
3601 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return() there.
3603 * Return: The updated value of @v.
3605 static __always_inline long
3606 atomic_long_inc_return(atomic_long_t *v)
3609 instrument_atomic_read_write(v, sizeof(*v));
3610 return raw_atomic_long_inc_return(v);
3614 * atomic_long_inc_return_acquire() - atomic increment with acquire ordering
3615 * @v: pointer to atomic_long_t
3617 * Atomically updates @v to (@v + 1) with acquire ordering.
3619 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_acquire() there.
3621 * Return: The updated value of @v.
3623 static __always_inline long
3624 atomic_long_inc_return_acquire(atomic_long_t *v)
3626 instrument_atomic_read_write(v, sizeof(*v));
3627 return raw_atomic_long_inc_return_acquire(v);
3631 * atomic_long_inc_return_release() - atomic increment with release ordering
3632 * @v: pointer to atomic_long_t
3634 * Atomically updates @v to (@v + 1) with release ordering.
3636 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_release() there.
3638 * Return: The updated value of @v.
3640 static __always_inline long
3641 atomic_long_inc_return_release(atomic_long_t *v)
3644 instrument_atomic_read_write(v, sizeof(*v));
3645 return raw_atomic_long_inc_return_release(v);
3649 * atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
3650 * @v: pointer to atomic_long_t
3652 * Atomically updates @v to (@v + 1) with relaxed ordering.
3654 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_relaxed() there.
3656 * Return: The updated value of @v.
3658 static __always_inline long
3659 atomic_long_inc_return_relaxed(atomic_long_t *v)
3661 instrument_atomic_read_write(v, sizeof(*v));
3662 return raw_atomic_long_inc_return_relaxed(v);
3666 * atomic_long_fetch_inc() - atomic increment with full ordering
3667 * @v: pointer to atomic_long_t
3669 * Atomically updates @v to (@v + 1) with full ordering.
3671 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc() there.
3673 * Return: The original value of @v.
3675 static __always_inline long
3676 atomic_long_fetch_inc(atomic_long_t *v)
3679 instrument_atomic_read_write(v, sizeof(*v));
3680 return raw_atomic_long_fetch_inc(v);
3684 * atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
3685 * @v: pointer to atomic_long_t
3687 * Atomically updates @v to (@v + 1) with acquire ordering.
3689 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_acquire() there.
3691 * Return: The original value of @v.
3693 static __always_inline long
3694 atomic_long_fetch_inc_acquire(atomic_long_t *v)
3696 instrument_atomic_read_write(v, sizeof(*v));
3697 return raw_atomic_long_fetch_inc_acquire(v);
3701 * atomic_long_fetch_inc_release() - atomic increment with release ordering
3702 * @v: pointer to atomic_long_t
3704 * Atomically updates @v to (@v + 1) with release ordering.
3706 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_release() there.
3708 * Return: The original value of @v.
3710 static __always_inline long
3711 atomic_long_fetch_inc_release(atomic_long_t *v)
3714 instrument_atomic_read_write(v, sizeof(*v));
3715 return raw_atomic_long_fetch_inc_release(v);
3719 * atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
3720 * @v: pointer to atomic_long_t
3722 * Atomically updates @v to (@v + 1) with relaxed ordering.
3724 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_relaxed() there.
3726 * Return: The original value of @v.
3728 static __always_inline long
3729 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
3731 instrument_atomic_read_write(v, sizeof(*v));
3732 return raw_atomic_long_fetch_inc_relaxed(v);
3736 * atomic_long_dec() - atomic decrement with relaxed ordering
3737 * @v: pointer to atomic_long_t
3739 * Atomically updates @v to (@v - 1) with relaxed ordering.
3741 * Unsafe to use in noinstr code; use raw_atomic_long_dec() there.
3745 static __always_inline void
3746 atomic_long_dec(atomic_long_t *v)
3748 instrument_atomic_read_write(v, sizeof(*v));
3749 raw_atomic_long_dec(v);
3753 * atomic_long_dec_return() - atomic decrement with full ordering
3754 * @v: pointer to atomic_long_t
3756 * Atomically updates @v to (@v - 1) with full ordering.
3758 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return() there.
3760 * Return: The updated value of @v.
3762 static __always_inline long
3763 atomic_long_dec_return(atomic_long_t *v)
3766 instrument_atomic_read_write(v, sizeof(*v));
3767 return raw_atomic_long_dec_return(v);
3771 * atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
3772 * @v: pointer to atomic_long_t
3774 * Atomically updates @v to (@v - 1) with acquire ordering.
3776 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_acquire() there.
3778 * Return: The updated value of @v.
3780 static __always_inline long
3781 atomic_long_dec_return_acquire(atomic_long_t *v)
3783 instrument_atomic_read_write(v, sizeof(*v));
3784 return raw_atomic_long_dec_return_acquire(v);
3788 * atomic_long_dec_return_release() - atomic decrement with release ordering
3789 * @v: pointer to atomic_long_t
3791 * Atomically updates @v to (@v - 1) with release ordering.
3793 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_release() there.
3795 * Return: The updated value of @v.
3797 static __always_inline long
3798 atomic_long_dec_return_release(atomic_long_t *v)
3801 instrument_atomic_read_write(v, sizeof(*v));
3802 return raw_atomic_long_dec_return_release(v);
3806 * atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
3807 * @v: pointer to atomic_long_t
3809 * Atomically updates @v to (@v - 1) with relaxed ordering.
3811 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_relaxed() there.
3813 * Return: The updated value of @v.
3815 static __always_inline long
3816 atomic_long_dec_return_relaxed(atomic_long_t *v)
3818 instrument_atomic_read_write(v, sizeof(*v));
3819 return raw_atomic_long_dec_return_relaxed(v);
3823 * atomic_long_fetch_dec() - atomic decrement with full ordering
3824 * @v: pointer to atomic_long_t
3826 * Atomically updates @v to (@v - 1) with full ordering.
3828 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec() there.
3830 * Return: The original value of @v.
3832 static __always_inline long
3833 atomic_long_fetch_dec(atomic_long_t *v)
3836 instrument_atomic_read_write(v, sizeof(*v));
3837 return raw_atomic_long_fetch_dec(v);
3841 * atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
3842 * @v: pointer to atomic_long_t
3844 * Atomically updates @v to (@v - 1) with acquire ordering.
3846 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_acquire() there.
3848 * Return: The original value of @v.
3850 static __always_inline long
3851 atomic_long_fetch_dec_acquire(atomic_long_t *v)
3853 instrument_atomic_read_write(v, sizeof(*v));
3854 return raw_atomic_long_fetch_dec_acquire(v);
3858 * atomic_long_fetch_dec_release() - atomic decrement with release ordering
3859 * @v: pointer to atomic_long_t
3861 * Atomically updates @v to (@v - 1) with release ordering.
3863 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_release() there.
3865 * Return: The original value of @v.
3867 static __always_inline long
3868 atomic_long_fetch_dec_release(atomic_long_t *v)
3871 instrument_atomic_read_write(v, sizeof(*v));
3872 return raw_atomic_long_fetch_dec_release(v);
3876 * atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
3877 * @v: pointer to atomic_long_t
3879 * Atomically updates @v to (@v - 1) with relaxed ordering.
3881 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_relaxed() there.
3883 * Return: The original value of @v.
3885 static __always_inline long
3886 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
3888 instrument_atomic_read_write(v, sizeof(*v));
3889 return raw_atomic_long_fetch_dec_relaxed(v);
3893 * atomic_long_and() - atomic bitwise AND with relaxed ordering
3895 * @v: pointer to atomic_long_t
3897 * Atomically updates @v to (@v & @i) with relaxed ordering.
3899 * Unsafe to use in noinstr code; use raw_atomic_long_and() there.
3903 static __always_inline void
3904 atomic_long_and(long i, atomic_long_t *v)
3906 instrument_atomic_read_write(v, sizeof(*v));
3907 raw_atomic_long_and(i, v);
3911 * atomic_long_fetch_and() - atomic bitwise AND with full ordering
3913 * @v: pointer to atomic_long_t
3915 * Atomically updates @v to (@v & @i) with full ordering.
3917 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and() there.
3919 * Return: The original value of @v.
3921 static __always_inline long
3922 atomic_long_fetch_and(long i, atomic_long_t *v)
3925 instrument_atomic_read_write(v, sizeof(*v));
3926 return raw_atomic_long_fetch_and(i, v);
3930 * atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
3932 * @v: pointer to atomic_long_t
3934 * Atomically updates @v to (@v & @i) with acquire ordering.
3936 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_acquire() there.
3938 * Return: The original value of @v.
3940 static __always_inline long
3941 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
3943 instrument_atomic_read_write(v, sizeof(*v));
3944 return raw_atomic_long_fetch_and_acquire(i, v);
3948 * atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
3950 * @v: pointer to atomic_long_t
3952 * Atomically updates @v to (@v & @i) with release ordering.
3954 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_release() there.
3956 * Return: The original value of @v.
3958 static __always_inline long
3959 atomic_long_fetch_and_release(long i, atomic_long_t *v)
3962 instrument_atomic_read_write(v, sizeof(*v));
3963 return raw_atomic_long_fetch_and_release(i, v);
3967 * atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
3969 * @v: pointer to atomic_long_t
3971 * Atomically updates @v to (@v & @i) with relaxed ordering.
3973 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_relaxed() there.
3975 * Return: The original value of @v.
3977 static __always_inline long
3978 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
3980 instrument_atomic_read_write(v, sizeof(*v));
3981 return raw_atomic_long_fetch_and_relaxed(i, v);
3985 * atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
3987 * @v: pointer to atomic_long_t
3989 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3991 * Unsafe to use in noinstr code; use raw_atomic_long_andnot() there.
3995 static __always_inline void
3996 atomic_long_andnot(long i, atomic_long_t *v)
3998 instrument_atomic_read_write(v, sizeof(*v));
3999 raw_atomic_long_andnot(i, v);
4003 * atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
4005 * @v: pointer to atomic_long_t
4007 * Atomically updates @v to (@v & ~@i) with full ordering.
4009 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot() there.
4011 * Return: The original value of @v.
4013 static __always_inline long
4014 atomic_long_fetch_andnot(long i, atomic_long_t *v)
4017 instrument_atomic_read_write(v, sizeof(*v));
4018 return raw_atomic_long_fetch_andnot(i, v);
4022 * atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
4024 * @v: pointer to atomic_long_t
4026 * Atomically updates @v to (@v & ~@i) with acquire ordering.
4028 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_acquire() there.
4030 * Return: The original value of @v.
4032 static __always_inline long
4033 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
4035 instrument_atomic_read_write(v, sizeof(*v));
4036 return raw_atomic_long_fetch_andnot_acquire(i, v);
4040 * atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
4042 * @v: pointer to atomic_long_t
4044 * Atomically updates @v to (@v & ~@i) with release ordering.
4046 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_release() there.
4048 * Return: The original value of @v.
4050 static __always_inline long
4051 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
4054 instrument_atomic_read_write(v, sizeof(*v));
4055 return raw_atomic_long_fetch_andnot_release(i, v);
4059 * atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
4061 * @v: pointer to atomic_long_t
4063 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
4065 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_relaxed() there.
4067 * Return: The original value of @v.
4069 static __always_inline long
4070 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
4072 instrument_atomic_read_write(v, sizeof(*v));
4073 return raw_atomic_long_fetch_andnot_relaxed(i, v);
4077 * atomic_long_or() - atomic bitwise OR with relaxed ordering
4079 * @v: pointer to atomic_long_t
4081 * Atomically updates @v to (@v | @i) with relaxed ordering.
4083 * Unsafe to use in noinstr code; use raw_atomic_long_or() there.
4087 static __always_inline void
4088 atomic_long_or(long i, atomic_long_t *v)
4090 instrument_atomic_read_write(v, sizeof(*v));
4091 raw_atomic_long_or(i, v);
4095 * atomic_long_fetch_or() - atomic bitwise OR with full ordering
4097 * @v: pointer to atomic_long_t
4099 * Atomically updates @v to (@v | @i) with full ordering.
4101 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or() there.
4103 * Return: The original value of @v.
4105 static __always_inline long
4106 atomic_long_fetch_or(long i, atomic_long_t *v)
4109 instrument_atomic_read_write(v, sizeof(*v));
4110 return raw_atomic_long_fetch_or(i, v);
4114 * atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
4116 * @v: pointer to atomic_long_t
4118 * Atomically updates @v to (@v | @i) with acquire ordering.
4120 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_acquire() there.
4122 * Return: The original value of @v.
4124 static __always_inline long
4125 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
4127 instrument_atomic_read_write(v, sizeof(*v));
4128 return raw_atomic_long_fetch_or_acquire(i, v);
4132 * atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
4134 * @v: pointer to atomic_long_t
4136 * Atomically updates @v to (@v | @i) with release ordering.
4138 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_release() there.
4140 * Return: The original value of @v.
4142 static __always_inline long
4143 atomic_long_fetch_or_release(long i, atomic_long_t *v)
4146 instrument_atomic_read_write(v, sizeof(*v));
4147 return raw_atomic_long_fetch_or_release(i, v);
4151 * atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
4153 * @v: pointer to atomic_long_t
4155 * Atomically updates @v to (@v | @i) with relaxed ordering.
4157 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_relaxed() there.
4159 * Return: The original value of @v.
4161 static __always_inline long
4162 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
4164 instrument_atomic_read_write(v, sizeof(*v));
4165 return raw_atomic_long_fetch_or_relaxed(i, v);
4169 * atomic_long_xor() - atomic bitwise XOR with relaxed ordering
4171 * @v: pointer to atomic_long_t
4173 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
4175 * Unsafe to use in noinstr code; use raw_atomic_long_xor() there.
4179 static __always_inline void
4180 atomic_long_xor(long i, atomic_long_t *v)
4182 instrument_atomic_read_write(v, sizeof(*v));
4183 raw_atomic_long_xor(i, v);
4187 * atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
4189 * @v: pointer to atomic_long_t
4191 * Atomically updates @v to (@v ^ @i) with full ordering.
4193 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor() there.
4195 * Return: The original value of @v.
4197 static __always_inline long
4198 atomic_long_fetch_xor(long i, atomic_long_t *v)
4201 instrument_atomic_read_write(v, sizeof(*v));
4202 return raw_atomic_long_fetch_xor(i, v);
4206 * atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
4208 * @v: pointer to atomic_long_t
4210 * Atomically updates @v to (@v ^ @i) with acquire ordering.
4212 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_acquire() there.
4214 * Return: The original value of @v.
4216 static __always_inline long
4217 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
4219 instrument_atomic_read_write(v, sizeof(*v));
4220 return raw_atomic_long_fetch_xor_acquire(i, v);
4224 * atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
4226 * @v: pointer to atomic_long_t
4228 * Atomically updates @v to (@v ^ @i) with release ordering.
4230 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_release() there.
4232 * Return: The original value of @v.
4234 static __always_inline long
4235 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
4238 instrument_atomic_read_write(v, sizeof(*v));
4239 return raw_atomic_long_fetch_xor_release(i, v);
4243 * atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
4245 * @v: pointer to atomic_long_t
4247 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
4249 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_relaxed() there.
4251 * Return: The original value of @v.
4253 static __always_inline long
4254 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
4256 instrument_atomic_read_write(v, sizeof(*v));
4257 return raw_atomic_long_fetch_xor_relaxed(i, v);
4261 * atomic_long_xchg() - atomic exchange with full ordering
4262 * @v: pointer to atomic_long_t
4263 * @new: long value to assign
4265 * Atomically updates @v to @new with full ordering.
4267 * Unsafe to use in noinstr code; use raw_atomic_long_xchg() there.
4269 * Return: The original value of @v.
4271 static __always_inline long
4272 atomic_long_xchg(atomic_long_t *v, long new)
4275 instrument_atomic_read_write(v, sizeof(*v));
4276 return raw_atomic_long_xchg(v, new);
4280 * atomic_long_xchg_acquire() - atomic exchange with acquire ordering
4281 * @v: pointer to atomic_long_t
4282 * @new: long value to assign
4284 * Atomically updates @v to @new with acquire ordering.
4286 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_acquire() there.
4288 * Return: The original value of @v.
4290 static __always_inline long
4291 atomic_long_xchg_acquire(atomic_long_t *v, long new)
4293 instrument_atomic_read_write(v, sizeof(*v));
4294 return raw_atomic_long_xchg_acquire(v, new);
4298 * atomic_long_xchg_release() - atomic exchange with release ordering
4299 * @v: pointer to atomic_long_t
4300 * @new: long value to assign
4302 * Atomically updates @v to @new with release ordering.
4304 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_release() there.
4306 * Return: The original value of @v.
4308 static __always_inline long
4309 atomic_long_xchg_release(atomic_long_t *v, long new)
4312 instrument_atomic_read_write(v, sizeof(*v));
4313 return raw_atomic_long_xchg_release(v, new);
4317 * atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
4318 * @v: pointer to atomic_long_t
4319 * @new: long value to assign
4321 * Atomically updates @v to @new with relaxed ordering.
4323 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_relaxed() there.
4325 * Return: The original value of @v.
4327 static __always_inline long
4328 atomic_long_xchg_relaxed(atomic_long_t *v, long new)
4330 instrument_atomic_read_write(v, sizeof(*v));
4331 return raw_atomic_long_xchg_relaxed(v, new);
4335 * atomic_long_cmpxchg() - atomic compare and exchange with full ordering
4336 * @v: pointer to atomic_long_t
4337 * @old: long value to compare with
4338 * @new: long value to assign
4340 * If (@v == @old), atomically updates @v to @new with full ordering.
4341 * Otherwise, @v is not modified and relaxed ordering is provided.
4343 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg() there.
4345 * Return: The original value of @v.
4347 static __always_inline long
4348 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
4351 instrument_atomic_read_write(v, sizeof(*v));
4352 return raw_atomic_long_cmpxchg(v, old, new);
4356 * atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4357 * @v: pointer to atomic_long_t
4358 * @old: long value to compare with
4359 * @new: long value to assign
4361 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4362 * Otherwise, @v is not modified and relaxed ordering is provided.
4364 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_acquire() there.
4366 * Return: The original value of @v.
4368 static __always_inline long
4369 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
4371 instrument_atomic_read_write(v, sizeof(*v));
4372 return raw_atomic_long_cmpxchg_acquire(v, old, new);
4376 * atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
4377 * @v: pointer to atomic_long_t
4378 * @old: long value to compare with
4379 * @new: long value to assign
4381 * If (@v == @old), atomically updates @v to @new with release ordering.
4382 * Otherwise, @v is not modified and relaxed ordering is provided.
4384 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_release() there.
4386 * Return: The original value of @v.
4388 static __always_inline long
4389 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
4392 instrument_atomic_read_write(v, sizeof(*v));
4393 return raw_atomic_long_cmpxchg_release(v, old, new);
4397 * atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4398 * @v: pointer to atomic_long_t
4399 * @old: long value to compare with
4400 * @new: long value to assign
4402 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4403 * Otherwise, @v is not modified and relaxed ordering is provided.
4405 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_relaxed() there.
4407 * Return: The original value of @v.
4409 static __always_inline long
4410 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
4412 instrument_atomic_read_write(v, sizeof(*v));
4413 return raw_atomic_long_cmpxchg_relaxed(v, old, new);
4417 * atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
4418 * @v: pointer to atomic_long_t
4419 * @old: pointer to long value to compare with
4420 * @new: long value to assign
4422 * If (@v == @old), atomically updates @v to @new with full ordering.
4423 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4424 * and relaxed ordering is provided.
4426 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there.
4428 * Return: @true if the exchange occured, @false otherwise.
4430 static __always_inline bool
4431 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
4434 instrument_atomic_read_write(v, sizeof(*v));
4435 instrument_atomic_read_write(old, sizeof(*old));
4436 return raw_atomic_long_try_cmpxchg(v, old, new);
4440 * atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4441 * @v: pointer to atomic_long_t
4442 * @old: pointer to long value to compare with
4443 * @new: long value to assign
4445 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4446 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4447 * and relaxed ordering is provided.
4449 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there.
4451 * Return: @true if the exchange occured, @false otherwise.
4453 static __always_inline bool
4454 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
4456 instrument_atomic_read_write(v, sizeof(*v));
4457 instrument_atomic_read_write(old, sizeof(*old));
4458 return raw_atomic_long_try_cmpxchg_acquire(v, old, new);
4462 * atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
4463 * @v: pointer to atomic_long_t
4464 * @old: pointer to long value to compare with
4465 * @new: long value to assign
4467 * If (@v == @old), atomically updates @v to @new with release ordering.
4468 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4469 * and relaxed ordering is provided.
4471 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there.
4473 * Return: @true if the exchange occured, @false otherwise.
4475 static __always_inline bool
4476 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
4479 instrument_atomic_read_write(v, sizeof(*v));
4480 instrument_atomic_read_write(old, sizeof(*old));
4481 return raw_atomic_long_try_cmpxchg_release(v, old, new);
4485 * atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4486 * @v: pointer to atomic_long_t
4487 * @old: pointer to long value to compare with
4488 * @new: long value to assign
4490 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4491 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4492 * and relaxed ordering is provided.
4494 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there.
4496 * Return: @true if the exchange occured, @false otherwise.
4498 static __always_inline bool
4499 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
4501 instrument_atomic_read_write(v, sizeof(*v));
4502 instrument_atomic_read_write(old, sizeof(*old));
4503 return raw_atomic_long_try_cmpxchg_relaxed(v, old, new);
4507 * atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
4508 * @i: long value to subtract
4509 * @v: pointer to atomic_long_t
4511 * Atomically updates @v to (@v - @i) with full ordering.
4513 * Unsafe to use in noinstr code; use raw_atomic_long_sub_and_test() there.
4515 * Return: @true if the resulting value of @v is zero, @false otherwise.
4517 static __always_inline bool
4518 atomic_long_sub_and_test(long i, atomic_long_t *v)
4521 instrument_atomic_read_write(v, sizeof(*v));
4522 return raw_atomic_long_sub_and_test(i, v);
4526 * atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
4527 * @v: pointer to atomic_long_t
4529 * Atomically updates @v to (@v - 1) with full ordering.
4531 * Unsafe to use in noinstr code; use raw_atomic_long_dec_and_test() there.
4533 * Return: @true if the resulting value of @v is zero, @false otherwise.
4535 static __always_inline bool
4536 atomic_long_dec_and_test(atomic_long_t *v)
4539 instrument_atomic_read_write(v, sizeof(*v));
4540 return raw_atomic_long_dec_and_test(v);
4544 * atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
4545 * @v: pointer to atomic_long_t
4547 * Atomically updates @v to (@v + 1) with full ordering.
4549 * Unsafe to use in noinstr code; use raw_atomic_long_inc_and_test() there.
4551 * Return: @true if the resulting value of @v is zero, @false otherwise.
4553 static __always_inline bool
4554 atomic_long_inc_and_test(atomic_long_t *v)
4557 instrument_atomic_read_write(v, sizeof(*v));
4558 return raw_atomic_long_inc_and_test(v);
4562 * atomic_long_add_negative() - atomic add and test if negative with full ordering
4563 * @i: long value to add
4564 * @v: pointer to atomic_long_t
4566 * Atomically updates @v to (@v + @i) with full ordering.
4568 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative() there.
4570 * Return: @true if the resulting value of @v is negative, @false otherwise.
4572 static __always_inline bool
4573 atomic_long_add_negative(long i, atomic_long_t *v)
4576 instrument_atomic_read_write(v, sizeof(*v));
4577 return raw_atomic_long_add_negative(i, v);
4581 * atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
4582 * @i: long value to add
4583 * @v: pointer to atomic_long_t
4585 * Atomically updates @v to (@v + @i) with acquire ordering.
4587 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_acquire() there.
4589 * Return: @true if the resulting value of @v is negative, @false otherwise.
4591 static __always_inline bool
4592 atomic_long_add_negative_acquire(long i, atomic_long_t *v)
4594 instrument_atomic_read_write(v, sizeof(*v));
4595 return raw_atomic_long_add_negative_acquire(i, v);
4599 * atomic_long_add_negative_release() - atomic add and test if negative with release ordering
4600 * @i: long value to add
4601 * @v: pointer to atomic_long_t
4603 * Atomically updates @v to (@v + @i) with release ordering.
4605 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_release() there.
4607 * Return: @true if the resulting value of @v is negative, @false otherwise.
4609 static __always_inline bool
4610 atomic_long_add_negative_release(long i, atomic_long_t *v)
4613 instrument_atomic_read_write(v, sizeof(*v));
4614 return raw_atomic_long_add_negative_release(i, v);
4618 * atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
4619 * @i: long value to add
4620 * @v: pointer to atomic_long_t
4622 * Atomically updates @v to (@v + @i) with relaxed ordering.
4624 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_relaxed() there.
4626 * Return: @true if the resulting value of @v is negative, @false otherwise.
4628 static __always_inline bool
4629 atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
4631 instrument_atomic_read_write(v, sizeof(*v));
4632 return raw_atomic_long_add_negative_relaxed(i, v);
4636 * atomic_long_fetch_add_unless() - atomic add unless value with full ordering
4637 * @v: pointer to atomic_long_t
4638 * @a: long value to add
4639 * @u: long value to compare with
4641 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4642 * Otherwise, @v is not modified and relaxed ordering is provided.
4644 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_unless() there.
4646 * Return: The original value of @v.
4648 static __always_inline long
4649 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
4652 instrument_atomic_read_write(v, sizeof(*v));
4653 return raw_atomic_long_fetch_add_unless(v, a, u);
4657 * atomic_long_add_unless() - atomic add unless value with full ordering
4658 * @v: pointer to atomic_long_t
4659 * @a: long value to add
4660 * @u: long value to compare with
4662 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4663 * Otherwise, @v is not modified and relaxed ordering is provided.
4665 * Unsafe to use in noinstr code; use raw_atomic_long_add_unless() there.
4667 * Return: @true if @v was updated, @false otherwise.
4669 static __always_inline bool
4670 atomic_long_add_unless(atomic_long_t *v, long a, long u)
4673 instrument_atomic_read_write(v, sizeof(*v));
4674 return raw_atomic_long_add_unless(v, a, u);
4678 * atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
4679 * @v: pointer to atomic_long_t
4681 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
4682 * Otherwise, @v is not modified and relaxed ordering is provided.
4684 * Unsafe to use in noinstr code; use raw_atomic_long_inc_not_zero() there.
4686 * Return: @true if @v was updated, @false otherwise.
4688 static __always_inline bool
4689 atomic_long_inc_not_zero(atomic_long_t *v)
4692 instrument_atomic_read_write(v, sizeof(*v));
4693 return raw_atomic_long_inc_not_zero(v);
4697 * atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
4698 * @v: pointer to atomic_long_t
4700 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
4701 * Otherwise, @v is not modified and relaxed ordering is provided.
4703 * Unsafe to use in noinstr code; use raw_atomic_long_inc_unless_negative() there.
4705 * Return: @true if @v was updated, @false otherwise.
4707 static __always_inline bool
4708 atomic_long_inc_unless_negative(atomic_long_t *v)
4711 instrument_atomic_read_write(v, sizeof(*v));
4712 return raw_atomic_long_inc_unless_negative(v);
4716 * atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
4717 * @v: pointer to atomic_long_t
4719 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
4720 * Otherwise, @v is not modified and relaxed ordering is provided.
4722 * Unsafe to use in noinstr code; use raw_atomic_long_dec_unless_positive() there.
4724 * Return: @true if @v was updated, @false otherwise.
4726 static __always_inline bool
4727 atomic_long_dec_unless_positive(atomic_long_t *v)
4730 instrument_atomic_read_write(v, sizeof(*v));
4731 return raw_atomic_long_dec_unless_positive(v);
4735 * atomic_long_dec_if_positive() - atomic decrement if positive with full ordering
4736 * @v: pointer to atomic_long_t
4738 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
4739 * Otherwise, @v is not modified and relaxed ordering is provided.
4741 * Unsafe to use in noinstr code; use raw_atomic_long_dec_if_positive() there.
4743 * Return: The old value of (@v - 1), regardless of whether @v was updated.
4745 static __always_inline long
4746 atomic_long_dec_if_positive(atomic_long_t *v)
4749 instrument_atomic_read_write(v, sizeof(*v));
4750 return raw_atomic_long_dec_if_positive(v);
4753 #define xchg(ptr, ...) \
4755 typeof(ptr) __ai_ptr = (ptr); \
4757 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4758 raw_xchg(__ai_ptr, __VA_ARGS__); \
4761 #define xchg_acquire(ptr, ...) \
4763 typeof(ptr) __ai_ptr = (ptr); \
4764 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4765 raw_xchg_acquire(__ai_ptr, __VA_ARGS__); \
4768 #define xchg_release(ptr, ...) \
4770 typeof(ptr) __ai_ptr = (ptr); \
4772 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4773 raw_xchg_release(__ai_ptr, __VA_ARGS__); \
4776 #define xchg_relaxed(ptr, ...) \
4778 typeof(ptr) __ai_ptr = (ptr); \
4779 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4780 raw_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
4783 #define cmpxchg(ptr, ...) \
4785 typeof(ptr) __ai_ptr = (ptr); \
4787 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4788 raw_cmpxchg(__ai_ptr, __VA_ARGS__); \
4791 #define cmpxchg_acquire(ptr, ...) \
4793 typeof(ptr) __ai_ptr = (ptr); \
4794 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4795 raw_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
4798 #define cmpxchg_release(ptr, ...) \
4800 typeof(ptr) __ai_ptr = (ptr); \
4802 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4803 raw_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
4806 #define cmpxchg_relaxed(ptr, ...) \
4808 typeof(ptr) __ai_ptr = (ptr); \
4809 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4810 raw_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
4813 #define cmpxchg64(ptr, ...) \
4815 typeof(ptr) __ai_ptr = (ptr); \
4817 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4818 raw_cmpxchg64(__ai_ptr, __VA_ARGS__); \
4821 #define cmpxchg64_acquire(ptr, ...) \
4823 typeof(ptr) __ai_ptr = (ptr); \
4824 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4825 raw_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
4828 #define cmpxchg64_release(ptr, ...) \
4830 typeof(ptr) __ai_ptr = (ptr); \
4832 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4833 raw_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
4836 #define cmpxchg64_relaxed(ptr, ...) \
4838 typeof(ptr) __ai_ptr = (ptr); \
4839 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4840 raw_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
4843 #define cmpxchg128(ptr, ...) \
4845 typeof(ptr) __ai_ptr = (ptr); \
4847 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4848 raw_cmpxchg128(__ai_ptr, __VA_ARGS__); \
4851 #define cmpxchg128_acquire(ptr, ...) \
4853 typeof(ptr) __ai_ptr = (ptr); \
4854 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4855 raw_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \
4858 #define cmpxchg128_release(ptr, ...) \
4860 typeof(ptr) __ai_ptr = (ptr); \
4862 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4863 raw_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \
4866 #define cmpxchg128_relaxed(ptr, ...) \
4868 typeof(ptr) __ai_ptr = (ptr); \
4869 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4870 raw_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \
4873 #define try_cmpxchg(ptr, oldp, ...) \
4875 typeof(ptr) __ai_ptr = (ptr); \
4876 typeof(oldp) __ai_oldp = (oldp); \
4878 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4879 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4880 raw_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4883 #define try_cmpxchg_acquire(ptr, oldp, ...) \
4885 typeof(ptr) __ai_ptr = (ptr); \
4886 typeof(oldp) __ai_oldp = (oldp); \
4887 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4888 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4889 raw_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4892 #define try_cmpxchg_release(ptr, oldp, ...) \
4894 typeof(ptr) __ai_ptr = (ptr); \
4895 typeof(oldp) __ai_oldp = (oldp); \
4897 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4898 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4899 raw_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4902 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
4904 typeof(ptr) __ai_ptr = (ptr); \
4905 typeof(oldp) __ai_oldp = (oldp); \
4906 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4907 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4908 raw_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4911 #define try_cmpxchg64(ptr, oldp, ...) \
4913 typeof(ptr) __ai_ptr = (ptr); \
4914 typeof(oldp) __ai_oldp = (oldp); \
4916 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4917 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4918 raw_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4921 #define try_cmpxchg64_acquire(ptr, oldp, ...) \
4923 typeof(ptr) __ai_ptr = (ptr); \
4924 typeof(oldp) __ai_oldp = (oldp); \
4925 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4926 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4927 raw_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4930 #define try_cmpxchg64_release(ptr, oldp, ...) \
4932 typeof(ptr) __ai_ptr = (ptr); \
4933 typeof(oldp) __ai_oldp = (oldp); \
4935 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4936 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4937 raw_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4940 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
4942 typeof(ptr) __ai_ptr = (ptr); \
4943 typeof(oldp) __ai_oldp = (oldp); \
4944 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4945 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4946 raw_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4949 #define try_cmpxchg128(ptr, oldp, ...) \
4951 typeof(ptr) __ai_ptr = (ptr); \
4952 typeof(oldp) __ai_oldp = (oldp); \
4954 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4955 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4956 raw_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4959 #define try_cmpxchg128_acquire(ptr, oldp, ...) \
4961 typeof(ptr) __ai_ptr = (ptr); \
4962 typeof(oldp) __ai_oldp = (oldp); \
4963 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4964 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4965 raw_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4968 #define try_cmpxchg128_release(ptr, oldp, ...) \
4970 typeof(ptr) __ai_ptr = (ptr); \
4971 typeof(oldp) __ai_oldp = (oldp); \
4973 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4974 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4975 raw_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4978 #define try_cmpxchg128_relaxed(ptr, oldp, ...) \
4980 typeof(ptr) __ai_ptr = (ptr); \
4981 typeof(oldp) __ai_oldp = (oldp); \
4982 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4983 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4984 raw_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4987 #define cmpxchg_local(ptr, ...) \
4989 typeof(ptr) __ai_ptr = (ptr); \
4990 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4991 raw_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
4994 #define cmpxchg64_local(ptr, ...) \
4996 typeof(ptr) __ai_ptr = (ptr); \
4997 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4998 raw_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
5001 #define cmpxchg128_local(ptr, ...) \
5003 typeof(ptr) __ai_ptr = (ptr); \
5004 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5005 raw_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \
5008 #define sync_cmpxchg(ptr, ...) \
5010 typeof(ptr) __ai_ptr = (ptr); \
5012 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5013 raw_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
5016 #define try_cmpxchg_local(ptr, oldp, ...) \
5018 typeof(ptr) __ai_ptr = (ptr); \
5019 typeof(oldp) __ai_oldp = (oldp); \
5020 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5021 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
5022 raw_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
5025 #define try_cmpxchg64_local(ptr, oldp, ...) \
5027 typeof(ptr) __ai_ptr = (ptr); \
5028 typeof(oldp) __ai_oldp = (oldp); \
5029 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5030 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
5031 raw_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
5034 #define try_cmpxchg128_local(ptr, oldp, ...) \
5036 typeof(ptr) __ai_ptr = (ptr); \
5037 typeof(oldp) __ai_oldp = (oldp); \
5038 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5039 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
5040 raw_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
5043 #define sync_try_cmpxchg(ptr, ...) \
5045 typeof(ptr) __ai_ptr = (ptr); \
5047 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5048 raw_sync_try_cmpxchg(__ai_ptr, __VA_ARGS__); \
5052 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
5053 // 8829b337928e9508259079d32581775ececd415b