include/linux/atomic/atomic-instrumented.h

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
   4 // DO NOT MODIFY THIS FILE DIRECTLY
   5
   6 /*
   7  * This file provides wrappers with KASAN instrumentation for atomic operations.
   8  * To use this functionality an arch's atomic.h file needs to define all
   9  * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
  10  * this file at the end. This file provides atomic_read() that forwards to
  11  * arch_atomic_read() for actual atomic operation.
  12  * Note: if an arch atomic operation is implemented by means of other atomic
  13  * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
  14  * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
  15  * double instrumentation.
  16  */
  17 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
  18 #define _LINUX_ATOMIC_INSTRUMENTED_H
  19
  20 #include <linux/build_bug.h>
  21 #include <linux/compiler.h>
  22 #include <linux/instrumented.h>
  23
  24 static __always_inline int
  25 atomic_read(const atomic_t *v)
  26 {
  27         instrument_atomic_read(v, sizeof(*v));
  28         return arch_atomic_read(v);
  29 }
  30
  31 static __always_inline int
  32 atomic_read_acquire(const atomic_t *v)
  33 {
  34         instrument_atomic_read(v, sizeof(*v));
  35         return arch_atomic_read_acquire(v);
  36 }
  37
  38 static __always_inline void
  39 atomic_set(atomic_t *v, int i)
  40 {
  41         instrument_atomic_write(v, sizeof(*v));
  42         arch_atomic_set(v, i);
  43 }
  44
  45 static __always_inline void
  46 atomic_set_release(atomic_t *v, int i)
  47 {
  48         kcsan_release();
  49         instrument_atomic_write(v, sizeof(*v));
  50         arch_atomic_set_release(v, i);
  51 }
  52
  53 static __always_inline void
  54 atomic_add(int i, atomic_t *v)
  55 {
  56         instrument_atomic_read_write(v, sizeof(*v));
  57         arch_atomic_add(i, v);
  58 }
  59
  60 static __always_inline int
  61 atomic_add_return(int i, atomic_t *v)
  62 {
  63         kcsan_mb();
  64         instrument_atomic_read_write(v, sizeof(*v));
  65         return arch_atomic_add_return(i, v);
  66 }
  67
  68 static __always_inline int
  69 atomic_add_return_acquire(int i, atomic_t *v)
  70 {
  71         instrument_atomic_read_write(v, sizeof(*v));
  72         return arch_atomic_add_return_acquire(i, v);
  73 }
  74
  75 static __always_inline int
  76 atomic_add_return_release(int i, atomic_t *v)
  77 {
  78         kcsan_release();
  79         instrument_atomic_read_write(v, sizeof(*v));
  80         return arch_atomic_add_return_release(i, v);
  81 }
  82
  83 static __always_inline int
  84 atomic_add_return_relaxed(int i, atomic_t *v)
  85 {
  86         instrument_atomic_read_write(v, sizeof(*v));
  87         return arch_atomic_add_return_relaxed(i, v);
  88 }
  89
  90 static __always_inline int
  91 atomic_fetch_add(int i, atomic_t *v)
  92 {
  93         kcsan_mb();
  94         instrument_atomic_read_write(v, sizeof(*v));
  95         return arch_atomic_fetch_add(i, v);
  96 }
  97
  98 static __always_inline int
  99 atomic_fetch_add_acquire(int i, atomic_t *v)
 100 {
 101         instrument_atomic_read_write(v, sizeof(*v));
 102         return arch_atomic_fetch_add_acquire(i, v);
 103 }
 104
 105 static __always_inline int
 106 atomic_fetch_add_release(int i, atomic_t *v)
 107 {
 108         kcsan_release();
 109         instrument_atomic_read_write(v, sizeof(*v));
 110         return arch_atomic_fetch_add_release(i, v);
 111 }
 112
 113 static __always_inline int
 114 atomic_fetch_add_relaxed(int i, atomic_t *v)
 115 {
 116         instrument_atomic_read_write(v, sizeof(*v));
 117         return arch_atomic_fetch_add_relaxed(i, v);
 118 }
 119
 120 static __always_inline void
 121 atomic_sub(int i, atomic_t *v)
 122 {
 123         instrument_atomic_read_write(v, sizeof(*v));
 124         arch_atomic_sub(i, v);
 125 }
 126
 127 static __always_inline int
 128 atomic_sub_return(int i, atomic_t *v)
 129 {
 130         kcsan_mb();
 131         instrument_atomic_read_write(v, sizeof(*v));
 132         return arch_atomic_sub_return(i, v);
 133 }
 134
 135 static __always_inline int
 136 atomic_sub_return_acquire(int i, atomic_t *v)
 137 {
 138         instrument_atomic_read_write(v, sizeof(*v));
 139         return arch_atomic_sub_return_acquire(i, v);
 140 }
 141
 142 static __always_inline int
 143 atomic_sub_return_release(int i, atomic_t *v)
 144 {
 145         kcsan_release();
 146         instrument_atomic_read_write(v, sizeof(*v));
 147         return arch_atomic_sub_return_release(i, v);
 148 }
 149
 150 static __always_inline int
 151 atomic_sub_return_relaxed(int i, atomic_t *v)
 152 {
 153         instrument_atomic_read_write(v, sizeof(*v));
 154         return arch_atomic_sub_return_relaxed(i, v);
 155 }
 156
 157 static __always_inline int
 158 atomic_fetch_sub(int i, atomic_t *v)
 159 {
 160         kcsan_mb();
 161         instrument_atomic_read_write(v, sizeof(*v));
 162         return arch_atomic_fetch_sub(i, v);
 163 }
 164
 165 static __always_inline int
 166 atomic_fetch_sub_acquire(int i, atomic_t *v)
 167 {
 168         instrument_atomic_read_write(v, sizeof(*v));
 169         return arch_atomic_fetch_sub_acquire(i, v);
 170 }
 171
 172 static __always_inline int
 173 atomic_fetch_sub_release(int i, atomic_t *v)
 174 {
 175         kcsan_release();
 176         instrument_atomic_read_write(v, sizeof(*v));
 177         return arch_atomic_fetch_sub_release(i, v);
 178 }
 179
 180 static __always_inline int
 181 atomic_fetch_sub_relaxed(int i, atomic_t *v)
 182 {
 183         instrument_atomic_read_write(v, sizeof(*v));
 184         return arch_atomic_fetch_sub_relaxed(i, v);
 185 }
 186
 187 static __always_inline void
 188 atomic_inc(atomic_t *v)
 189 {
 190         instrument_atomic_read_write(v, sizeof(*v));
 191         arch_atomic_inc(v);
 192 }
 193
 194 static __always_inline int
 195 atomic_inc_return(atomic_t *v)
 196 {
 197         kcsan_mb();
 198         instrument_atomic_read_write(v, sizeof(*v));
 199         return arch_atomic_inc_return(v);
 200 }
 201
 202 static __always_inline int
 203 atomic_inc_return_acquire(atomic_t *v)
 204 {
 205         instrument_atomic_read_write(v, sizeof(*v));
 206         return arch_atomic_inc_return_acquire(v);
 207 }
 208
 209 static __always_inline int
 210 atomic_inc_return_release(atomic_t *v)
 211 {
 212         kcsan_release();
 213         instrument_atomic_read_write(v, sizeof(*v));
 214         return arch_atomic_inc_return_release(v);
 215 }
 216
 217 static __always_inline int
 218 atomic_inc_return_relaxed(atomic_t *v)
 219 {
 220         instrument_atomic_read_write(v, sizeof(*v));
 221         return arch_atomic_inc_return_relaxed(v);
 222 }
 223
 224 static __always_inline int
 225 atomic_fetch_inc(atomic_t *v)
 226 {
 227         kcsan_mb();
 228         instrument_atomic_read_write(v, sizeof(*v));
 229         return arch_atomic_fetch_inc(v);
 230 }
 231
 232 static __always_inline int
 233 atomic_fetch_inc_acquire(atomic_t *v)
 234 {
 235         instrument_atomic_read_write(v, sizeof(*v));
 236         return arch_atomic_fetch_inc_acquire(v);
 237 }
 238
 239 static __always_inline int
 240 atomic_fetch_inc_release(atomic_t *v)
 241 {
 242         kcsan_release();
 243         instrument_atomic_read_write(v, sizeof(*v));
 244         return arch_atomic_fetch_inc_release(v);
 245 }
 246
 247 static __always_inline int
 248 atomic_fetch_inc_relaxed(atomic_t *v)
 249 {
 250         instrument_atomic_read_write(v, sizeof(*v));
 251         return arch_atomic_fetch_inc_relaxed(v);
 252 }
 253
 254 static __always_inline void
 255 atomic_dec(atomic_t *v)
 256 {
 257         instrument_atomic_read_write(v, sizeof(*v));
 258         arch_atomic_dec(v);
 259 }
 260
 261 static __always_inline int
 262 atomic_dec_return(atomic_t *v)
 263 {
 264         kcsan_mb();
 265         instrument_atomic_read_write(v, sizeof(*v));
 266         return arch_atomic_dec_return(v);
 267 }
 268
 269 static __always_inline int
 270 atomic_dec_return_acquire(atomic_t *v)
 271 {
 272         instrument_atomic_read_write(v, sizeof(*v));
 273         return arch_atomic_dec_return_acquire(v);
 274 }
 275
 276 static __always_inline int
 277 atomic_dec_return_release(atomic_t *v)
 278 {
 279         kcsan_release();
 280         instrument_atomic_read_write(v, sizeof(*v));
 281         return arch_atomic_dec_return_release(v);
 282 }
 283
 284 static __always_inline int
 285 atomic_dec_return_relaxed(atomic_t *v)
 286 {
 287         instrument_atomic_read_write(v, sizeof(*v));
 288         return arch_atomic_dec_return_relaxed(v);
 289 }
 290
 291 static __always_inline int
 292 atomic_fetch_dec(atomic_t *v)
 293 {
 294         kcsan_mb();
 295         instrument_atomic_read_write(v, sizeof(*v));
 296         return arch_atomic_fetch_dec(v);
 297 }
 298
 299 static __always_inline int
 300 atomic_fetch_dec_acquire(atomic_t *v)
 301 {
 302         instrument_atomic_read_write(v, sizeof(*v));
 303         return arch_atomic_fetch_dec_acquire(v);
 304 }
 305
 306 static __always_inline int
 307 atomic_fetch_dec_release(atomic_t *v)
 308 {
 309         kcsan_release();
 310         instrument_atomic_read_write(v, sizeof(*v));
 311         return arch_atomic_fetch_dec_release(v);
 312 }
 313
 314 static __always_inline int
 315 atomic_fetch_dec_relaxed(atomic_t *v)
 316 {
 317         instrument_atomic_read_write(v, sizeof(*v));
 318         return arch_atomic_fetch_dec_relaxed(v);
 319 }
 320
 321 static __always_inline void
 322 atomic_and(int i, atomic_t *v)
 323 {
 324         instrument_atomic_read_write(v, sizeof(*v));
 325         arch_atomic_and(i, v);
 326 }
 327
 328 static __always_inline int
 329 atomic_fetch_and(int i, atomic_t *v)
 330 {
 331         kcsan_mb();
 332         instrument_atomic_read_write(v, sizeof(*v));
 333         return arch_atomic_fetch_and(i, v);
 334 }
 335
 336 static __always_inline int
 337 atomic_fetch_and_acquire(int i, atomic_t *v)
 338 {
 339         instrument_atomic_read_write(v, sizeof(*v));
 340         return arch_atomic_fetch_and_acquire(i, v);
 341 }
 342
 343 static __always_inline int
 344 atomic_fetch_and_release(int i, atomic_t *v)
 345 {
 346         kcsan_release();
 347         instrument_atomic_read_write(v, sizeof(*v));
 348         return arch_atomic_fetch_and_release(i, v);
 349 }
 350
 351 static __always_inline int
 352 atomic_fetch_and_relaxed(int i, atomic_t *v)
 353 {
 354         instrument_atomic_read_write(v, sizeof(*v));
 355         return arch_atomic_fetch_and_relaxed(i, v);
 356 }
 357
 358 static __always_inline void
 359 atomic_andnot(int i, atomic_t *v)
 360 {
 361         instrument_atomic_read_write(v, sizeof(*v));
 362         arch_atomic_andnot(i, v);
 363 }
 364
 365 static __always_inline int
 366 atomic_fetch_andnot(int i, atomic_t *v)
 367 {
 368         kcsan_mb();
 369         instrument_atomic_read_write(v, sizeof(*v));
 370         return arch_atomic_fetch_andnot(i, v);
 371 }
 372
 373 static __always_inline int
 374 atomic_fetch_andnot_acquire(int i, atomic_t *v)
 375 {
 376         instrument_atomic_read_write(v, sizeof(*v));
 377         return arch_atomic_fetch_andnot_acquire(i, v);
 378 }
 379
 380 static __always_inline int
 381 atomic_fetch_andnot_release(int i, atomic_t *v)
 382 {
 383         kcsan_release();
 384         instrument_atomic_read_write(v, sizeof(*v));
 385         return arch_atomic_fetch_andnot_release(i, v);
 386 }
 387
 388 static __always_inline int
 389 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
 390 {
 391         instrument_atomic_read_write(v, sizeof(*v));
 392         return arch_atomic_fetch_andnot_relaxed(i, v);
 393 }
 394
 395 static __always_inline void
 396 atomic_or(int i, atomic_t *v)
 397 {
 398         instrument_atomic_read_write(v, sizeof(*v));
 399         arch_atomic_or(i, v);
 400 }
 401
 402 static __always_inline int
 403 atomic_fetch_or(int i, atomic_t *v)
 404 {
 405         kcsan_mb();
 406         instrument_atomic_read_write(v, sizeof(*v));
 407         return arch_atomic_fetch_or(i, v);
 408 }
 409
 410 static __always_inline int
 411 atomic_fetch_or_acquire(int i, atomic_t *v)
 412 {
 413         instrument_atomic_read_write(v, sizeof(*v));
 414         return arch_atomic_fetch_or_acquire(i, v);
 415 }
 416
 417 static __always_inline int
 418 atomic_fetch_or_release(int i, atomic_t *v)
 419 {
 420         kcsan_release();
 421         instrument_atomic_read_write(v, sizeof(*v));
 422         return arch_atomic_fetch_or_release(i, v);
 423 }
 424
 425 static __always_inline int
 426 atomic_fetch_or_relaxed(int i, atomic_t *v)
 427 {
 428         instrument_atomic_read_write(v, sizeof(*v));
 429         return arch_atomic_fetch_or_relaxed(i, v);
 430 }
 431
 432 static __always_inline void
 433 atomic_xor(int i, atomic_t *v)
 434 {
 435         instrument_atomic_read_write(v, sizeof(*v));
 436         arch_atomic_xor(i, v);
 437 }
 438
 439 static __always_inline int
 440 atomic_fetch_xor(int i, atomic_t *v)
 441 {
 442         kcsan_mb();
 443         instrument_atomic_read_write(v, sizeof(*v));
 444         return arch_atomic_fetch_xor(i, v);
 445 }
 446
 447 static __always_inline int
 448 atomic_fetch_xor_acquire(int i, atomic_t *v)
 449 {
 450         instrument_atomic_read_write(v, sizeof(*v));
 451         return arch_atomic_fetch_xor_acquire(i, v);
 452 }
 453
 454 static __always_inline int
 455 atomic_fetch_xor_release(int i, atomic_t *v)
 456 {
 457         kcsan_release();
 458         instrument_atomic_read_write(v, sizeof(*v));
 459         return arch_atomic_fetch_xor_release(i, v);
 460 }
 461
 462 static __always_inline int
 463 atomic_fetch_xor_relaxed(int i, atomic_t *v)
 464 {
 465         instrument_atomic_read_write(v, sizeof(*v));
 466         return arch_atomic_fetch_xor_relaxed(i, v);
 467 }
 468
 469 static __always_inline int
 470 atomic_xchg(atomic_t *v, int i)
 471 {
 472         kcsan_mb();
 473         instrument_atomic_read_write(v, sizeof(*v));
 474         return arch_atomic_xchg(v, i);
 475 }
 476
 477 static __always_inline int
 478 atomic_xchg_acquire(atomic_t *v, int i)
 479 {
 480         instrument_atomic_read_write(v, sizeof(*v));
 481         return arch_atomic_xchg_acquire(v, i);
 482 }
 483
 484 static __always_inline int
 485 atomic_xchg_release(atomic_t *v, int i)
 486 {
 487         kcsan_release();
 488         instrument_atomic_read_write(v, sizeof(*v));
 489         return arch_atomic_xchg_release(v, i);
 490 }
 491
 492 static __always_inline int
 493 atomic_xchg_relaxed(atomic_t *v, int i)
 494 {
 495         instrument_atomic_read_write(v, sizeof(*v));
 496         return arch_atomic_xchg_relaxed(v, i);
 497 }
 498
 499 static __always_inline int
 500 atomic_cmpxchg(atomic_t *v, int old, int new)
 501 {
 502         kcsan_mb();
 503         instrument_atomic_read_write(v, sizeof(*v));
 504         return arch_atomic_cmpxchg(v, old, new);
 505 }
 506
 507 static __always_inline int
 508 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
 509 {
 510         instrument_atomic_read_write(v, sizeof(*v));
 511         return arch_atomic_cmpxchg_acquire(v, old, new);
 512 }
 513
 514 static __always_inline int
 515 atomic_cmpxchg_release(atomic_t *v, int old, int new)
 516 {
 517         kcsan_release();
 518         instrument_atomic_read_write(v, sizeof(*v));
 519         return arch_atomic_cmpxchg_release(v, old, new);
 520 }
 521
 522 static __always_inline int
 523 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
 524 {
 525         instrument_atomic_read_write(v, sizeof(*v));
 526         return arch_atomic_cmpxchg_relaxed(v, old, new);
 527 }
 528
 529 static __always_inline bool
 530 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
 531 {
 532         kcsan_mb();
 533         instrument_atomic_read_write(v, sizeof(*v));
 534         instrument_atomic_read_write(old, sizeof(*old));
 535         return arch_atomic_try_cmpxchg(v, old, new);
 536 }
 537
 538 static __always_inline bool
 539 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
 540 {
 541         instrument_atomic_read_write(v, sizeof(*v));
 542         instrument_atomic_read_write(old, sizeof(*old));
 543         return arch_atomic_try_cmpxchg_acquire(v, old, new);
 544 }
 545
 546 static __always_inline bool
 547 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
 548 {
 549         kcsan_release();
 550         instrument_atomic_read_write(v, sizeof(*v));
 551         instrument_atomic_read_write(old, sizeof(*old));
 552         return arch_atomic_try_cmpxchg_release(v, old, new);
 553 }
 554
 555 static __always_inline bool
 556 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
 557 {
 558         instrument_atomic_read_write(v, sizeof(*v));
 559         instrument_atomic_read_write(old, sizeof(*old));
 560         return arch_atomic_try_cmpxchg_relaxed(v, old, new);
 561 }
 562
 563 static __always_inline bool
 564 atomic_sub_and_test(int i, atomic_t *v)
 565 {
 566         kcsan_mb();
 567         instrument_atomic_read_write(v, sizeof(*v));
 568         return arch_atomic_sub_and_test(i, v);
 569 }
 570
 571 static __always_inline bool
 572 atomic_dec_and_test(atomic_t *v)
 573 {
 574         kcsan_mb();
 575         instrument_atomic_read_write(v, sizeof(*v));
 576         return arch_atomic_dec_and_test(v);
 577 }
 578
 579 static __always_inline bool
 580 atomic_inc_and_test(atomic_t *v)
 581 {
 582         kcsan_mb();
 583         instrument_atomic_read_write(v, sizeof(*v));
 584         return arch_atomic_inc_and_test(v);
 585 }
 586
 587 static __always_inline bool
 588 atomic_add_negative(int i, atomic_t *v)
 589 {
 590         kcsan_mb();
 591         instrument_atomic_read_write(v, sizeof(*v));
 592         return arch_atomic_add_negative(i, v);
 593 }
 594
 595 static __always_inline bool
 596 atomic_add_negative_acquire(int i, atomic_t *v)
 597 {
 598         instrument_atomic_read_write(v, sizeof(*v));
 599         return arch_atomic_add_negative_acquire(i, v);
 600 }
 601
 602 static __always_inline bool
 603 atomic_add_negative_release(int i, atomic_t *v)
 604 {
 605         kcsan_release();
 606         instrument_atomic_read_write(v, sizeof(*v));
 607         return arch_atomic_add_negative_release(i, v);
 608 }
 609
 610 static __always_inline bool
 611 atomic_add_negative_relaxed(int i, atomic_t *v)
 612 {
 613         instrument_atomic_read_write(v, sizeof(*v));
 614         return arch_atomic_add_negative_relaxed(i, v);
 615 }
 616
 617 static __always_inline int
 618 atomic_fetch_add_unless(atomic_t *v, int a, int u)
 619 {
 620         kcsan_mb();
 621         instrument_atomic_read_write(v, sizeof(*v));
 622         return arch_atomic_fetch_add_unless(v, a, u);
 623 }
 624
 625 static __always_inline bool
 626 atomic_add_unless(atomic_t *v, int a, int u)
 627 {
 628         kcsan_mb();
 629         instrument_atomic_read_write(v, sizeof(*v));
 630         return arch_atomic_add_unless(v, a, u);
 631 }
 632
 633 static __always_inline bool
 634 atomic_inc_not_zero(atomic_t *v)
 635 {
 636         kcsan_mb();
 637         instrument_atomic_read_write(v, sizeof(*v));
 638         return arch_atomic_inc_not_zero(v);
 639 }
 640
 641 static __always_inline bool
 642 atomic_inc_unless_negative(atomic_t *v)
 643 {
 644         kcsan_mb();
 645         instrument_atomic_read_write(v, sizeof(*v));
 646         return arch_atomic_inc_unless_negative(v);
 647 }
 648
 649 static __always_inline bool
 650 atomic_dec_unless_positive(atomic_t *v)
 651 {
 652         kcsan_mb();
 653         instrument_atomic_read_write(v, sizeof(*v));
 654         return arch_atomic_dec_unless_positive(v);
 655 }
 656
 657 static __always_inline int
 658 atomic_dec_if_positive(atomic_t *v)
 659 {
 660         kcsan_mb();
 661         instrument_atomic_read_write(v, sizeof(*v));
 662         return arch_atomic_dec_if_positive(v);
 663 }
 664
 665 static __always_inline s64
 666 atomic64_read(const atomic64_t *v)
 667 {
 668         instrument_atomic_read(v, sizeof(*v));
 669         return arch_atomic64_read(v);
 670 }
 671
 672 static __always_inline s64
 673 atomic64_read_acquire(const atomic64_t *v)
 674 {
 675         instrument_atomic_read(v, sizeof(*v));
 676         return arch_atomic64_read_acquire(v);
 677 }
 678
 679 static __always_inline void
 680 atomic64_set(atomic64_t *v, s64 i)
 681 {
 682         instrument_atomic_write(v, sizeof(*v));
 683         arch_atomic64_set(v, i);
 684 }
 685
 686 static __always_inline void
 687 atomic64_set_release(atomic64_t *v, s64 i)
 688 {
 689         kcsan_release();
 690         instrument_atomic_write(v, sizeof(*v));
 691         arch_atomic64_set_release(v, i);
 692 }
 693
 694 static __always_inline void
 695 atomic64_add(s64 i, atomic64_t *v)
 696 {
 697         instrument_atomic_read_write(v, sizeof(*v));
 698         arch_atomic64_add(i, v);
 699 }
 700
 701 static __always_inline s64
 702 atomic64_add_return(s64 i, atomic64_t *v)
 703 {
 704         kcsan_mb();
 705         instrument_atomic_read_write(v, sizeof(*v));
 706         return arch_atomic64_add_return(i, v);
 707 }
 708
 709 static __always_inline s64
 710 atomic64_add_return_acquire(s64 i, atomic64_t *v)
 711 {
 712         instrument_atomic_read_write(v, sizeof(*v));
 713         return arch_atomic64_add_return_acquire(i, v);
 714 }
 715
 716 static __always_inline s64
 717 atomic64_add_return_release(s64 i, atomic64_t *v)
 718 {
 719         kcsan_release();
 720         instrument_atomic_read_write(v, sizeof(*v));
 721         return arch_atomic64_add_return_release(i, v);
 722 }
 723
 724 static __always_inline s64
 725 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
 726 {
 727         instrument_atomic_read_write(v, sizeof(*v));
 728         return arch_atomic64_add_return_relaxed(i, v);
 729 }
 730
 731 static __always_inline s64
 732 atomic64_fetch_add(s64 i, atomic64_t *v)
 733 {
 734         kcsan_mb();
 735         instrument_atomic_read_write(v, sizeof(*v));
 736         return arch_atomic64_fetch_add(i, v);
 737 }
 738
 739 static __always_inline s64
 740 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
 741 {
 742         instrument_atomic_read_write(v, sizeof(*v));
 743         return arch_atomic64_fetch_add_acquire(i, v);
 744 }
 745
 746 static __always_inline s64
 747 atomic64_fetch_add_release(s64 i, atomic64_t *v)
 748 {
 749         kcsan_release();
 750         instrument_atomic_read_write(v, sizeof(*v));
 751         return arch_atomic64_fetch_add_release(i, v);
 752 }
 753
 754 static __always_inline s64
 755 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
 756 {
 757         instrument_atomic_read_write(v, sizeof(*v));
 758         return arch_atomic64_fetch_add_relaxed(i, v);
 759 }
 760
 761 static __always_inline void
 762 atomic64_sub(s64 i, atomic64_t *v)
 763 {
 764         instrument_atomic_read_write(v, sizeof(*v));
 765         arch_atomic64_sub(i, v);
 766 }
 767
 768 static __always_inline s64
 769 atomic64_sub_return(s64 i, atomic64_t *v)
 770 {
 771         kcsan_mb();
 772         instrument_atomic_read_write(v, sizeof(*v));
 773         return arch_atomic64_sub_return(i, v);
 774 }
 775
 776 static __always_inline s64
 777 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
 778 {
 779         instrument_atomic_read_write(v, sizeof(*v));
 780         return arch_atomic64_sub_return_acquire(i, v);
 781 }
 782
 783 static __always_inline s64
 784 atomic64_sub_return_release(s64 i, atomic64_t *v)
 785 {
 786         kcsan_release();
 787         instrument_atomic_read_write(v, sizeof(*v));
 788         return arch_atomic64_sub_return_release(i, v);
 789 }
 790
 791 static __always_inline s64
 792 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
 793 {
 794         instrument_atomic_read_write(v, sizeof(*v));
 795         return arch_atomic64_sub_return_relaxed(i, v);
 796 }
 797
 798 static __always_inline s64
 799 atomic64_fetch_sub(s64 i, atomic64_t *v)
 800 {
 801         kcsan_mb();
 802         instrument_atomic_read_write(v, sizeof(*v));
 803         return arch_atomic64_fetch_sub(i, v);
 804 }
 805
 806 static __always_inline s64
 807 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
 808 {
 809         instrument_atomic_read_write(v, sizeof(*v));
 810         return arch_atomic64_fetch_sub_acquire(i, v);
 811 }
 812
 813 static __always_inline s64
 814 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
 815 {
 816         kcsan_release();
 817         instrument_atomic_read_write(v, sizeof(*v));
 818         return arch_atomic64_fetch_sub_release(i, v);
 819 }
 820
 821 static __always_inline s64
 822 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
 823 {
 824         instrument_atomic_read_write(v, sizeof(*v));
 825         return arch_atomic64_fetch_sub_relaxed(i, v);
 826 }
 827
 828 static __always_inline void
 829 atomic64_inc(atomic64_t *v)
 830 {
 831         instrument_atomic_read_write(v, sizeof(*v));
 832         arch_atomic64_inc(v);
 833 }
 834
 835 static __always_inline s64
 836 atomic64_inc_return(atomic64_t *v)
 837 {
 838         kcsan_mb();
 839         instrument_atomic_read_write(v, sizeof(*v));
 840         return arch_atomic64_inc_return(v);
 841 }
 842
 843 static __always_inline s64
 844 atomic64_inc_return_acquire(atomic64_t *v)
 845 {
 846         instrument_atomic_read_write(v, sizeof(*v));
 847         return arch_atomic64_inc_return_acquire(v);
 848 }
 849
 850 static __always_inline s64
 851 atomic64_inc_return_release(atomic64_t *v)
 852 {
 853         kcsan_release();
 854         instrument_atomic_read_write(v, sizeof(*v));
 855         return arch_atomic64_inc_return_release(v);
 856 }
 857
 858 static __always_inline s64
 859 atomic64_inc_return_relaxed(atomic64_t *v)
 860 {
 861         instrument_atomic_read_write(v, sizeof(*v));
 862         return arch_atomic64_inc_return_relaxed(v);
 863 }
 864
 865 static __always_inline s64
 866 atomic64_fetch_inc(atomic64_t *v)
 867 {
 868         kcsan_mb();
 869         instrument_atomic_read_write(v, sizeof(*v));
 870         return arch_atomic64_fetch_inc(v);
 871 }
 872
 873 static __always_inline s64
 874 atomic64_fetch_inc_acquire(atomic64_t *v)
 875 {
 876         instrument_atomic_read_write(v, sizeof(*v));
 877         return arch_atomic64_fetch_inc_acquire(v);
 878 }
 879
 880 static __always_inline s64
 881 atomic64_fetch_inc_release(atomic64_t *v)
 882 {
 883         kcsan_release();
 884         instrument_atomic_read_write(v, sizeof(*v));
 885         return arch_atomic64_fetch_inc_release(v);
 886 }
 887
 888 static __always_inline s64
 889 atomic64_fetch_inc_relaxed(atomic64_t *v)
 890 {
 891         instrument_atomic_read_write(v, sizeof(*v));
 892         return arch_atomic64_fetch_inc_relaxed(v);
 893 }
 894
 895 static __always_inline void
 896 atomic64_dec(atomic64_t *v)
 897 {
 898         instrument_atomic_read_write(v, sizeof(*v));
 899         arch_atomic64_dec(v);
 900 }
 901
 902 static __always_inline s64
 903 atomic64_dec_return(atomic64_t *v)
 904 {
 905         kcsan_mb();
 906         instrument_atomic_read_write(v, sizeof(*v));
 907         return arch_atomic64_dec_return(v);
 908 }
 909
 910 static __always_inline s64
 911 atomic64_dec_return_acquire(atomic64_t *v)
 912 {
 913         instrument_atomic_read_write(v, sizeof(*v));
 914         return arch_atomic64_dec_return_acquire(v);
 915 }
 916
 917 static __always_inline s64
 918 atomic64_dec_return_release(atomic64_t *v)
 919 {
 920         kcsan_release();
 921         instrument_atomic_read_write(v, sizeof(*v));
 922         return arch_atomic64_dec_return_release(v);
 923 }
 924
 925 static __always_inline s64
 926 atomic64_dec_return_relaxed(atomic64_t *v)
 927 {
 928         instrument_atomic_read_write(v, sizeof(*v));
 929         return arch_atomic64_dec_return_relaxed(v);
 930 }
 931
 932 static __always_inline s64
 933 atomic64_fetch_dec(atomic64_t *v)
 934 {
 935         kcsan_mb();
 936         instrument_atomic_read_write(v, sizeof(*v));
 937         return arch_atomic64_fetch_dec(v);
 938 }
 939
 940 static __always_inline s64
 941 atomic64_fetch_dec_acquire(atomic64_t *v)
 942 {
 943         instrument_atomic_read_write(v, sizeof(*v));
 944         return arch_atomic64_fetch_dec_acquire(v);
 945 }
 946
 947 static __always_inline s64
 948 atomic64_fetch_dec_release(atomic64_t *v)
 949 {
 950         kcsan_release();
 951         instrument_atomic_read_write(v, sizeof(*v));
 952         return arch_atomic64_fetch_dec_release(v);
 953 }
 954
 955 static __always_inline s64
 956 atomic64_fetch_dec_relaxed(atomic64_t *v)
 957 {
 958         instrument_atomic_read_write(v, sizeof(*v));
 959         return arch_atomic64_fetch_dec_relaxed(v);
 960 }
 961
 962 static __always_inline void
 963 atomic64_and(s64 i, atomic64_t *v)
 964 {
 965         instrument_atomic_read_write(v, sizeof(*v));
 966         arch_atomic64_and(i, v);
 967 }
 968
 969 static __always_inline s64
 970 atomic64_fetch_and(s64 i, atomic64_t *v)
 971 {
 972         kcsan_mb();
 973         instrument_atomic_read_write(v, sizeof(*v));
 974         return arch_atomic64_fetch_and(i, v);
 975 }
 976
 977 static __always_inline s64
 978 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
 979 {
 980         instrument_atomic_read_write(v, sizeof(*v));
 981         return arch_atomic64_fetch_and_acquire(i, v);
 982 }
 983
 984 static __always_inline s64
 985 atomic64_fetch_and_release(s64 i, atomic64_t *v)
 986 {
 987         kcsan_release();
 988         instrument_atomic_read_write(v, sizeof(*v));
 989         return arch_atomic64_fetch_and_release(i, v);
 990 }
 991
 992 static __always_inline s64
 993 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
 994 {
 995         instrument_atomic_read_write(v, sizeof(*v));
 996         return arch_atomic64_fetch_and_relaxed(i, v);
 997 }
 998
 999 static __always_inline void
1000 atomic64_andnot(s64 i, atomic64_t *v)
1001 {
1002         instrument_atomic_read_write(v, sizeof(*v));
1003         arch_atomic64_andnot(i, v);
1004 }
1005
1006 static __always_inline s64
1007 atomic64_fetch_andnot(s64 i, atomic64_t *v)
1008 {
1009         kcsan_mb();
1010         instrument_atomic_read_write(v, sizeof(*v));
1011         return arch_atomic64_fetch_andnot(i, v);
1012 }
1013
1014 static __always_inline s64
1015 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1016 {
1017         instrument_atomic_read_write(v, sizeof(*v));
1018         return arch_atomic64_fetch_andnot_acquire(i, v);
1019 }
1020
1021 static __always_inline s64
1022 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1023 {
1024         kcsan_release();
1025         instrument_atomic_read_write(v, sizeof(*v));
1026         return arch_atomic64_fetch_andnot_release(i, v);
1027 }
1028
1029 static __always_inline s64
1030 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1031 {
1032         instrument_atomic_read_write(v, sizeof(*v));
1033         return arch_atomic64_fetch_andnot_relaxed(i, v);
1034 }
1035
1036 static __always_inline void
1037 atomic64_or(s64 i, atomic64_t *v)
1038 {
1039         instrument_atomic_read_write(v, sizeof(*v));
1040         arch_atomic64_or(i, v);
1041 }
1042
1043 static __always_inline s64
1044 atomic64_fetch_or(s64 i, atomic64_t *v)
1045 {
1046         kcsan_mb();
1047         instrument_atomic_read_write(v, sizeof(*v));
1048         return arch_atomic64_fetch_or(i, v);
1049 }
1050
1051 static __always_inline s64
1052 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1053 {
1054         instrument_atomic_read_write(v, sizeof(*v));
1055         return arch_atomic64_fetch_or_acquire(i, v);
1056 }
1057
1058 static __always_inline s64
1059 atomic64_fetch_or_release(s64 i, atomic64_t *v)
1060 {
1061         kcsan_release();
1062         instrument_atomic_read_write(v, sizeof(*v));
1063         return arch_atomic64_fetch_or_release(i, v);
1064 }
1065
1066 static __always_inline s64
1067 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1068 {
1069         instrument_atomic_read_write(v, sizeof(*v));
1070         return arch_atomic64_fetch_or_relaxed(i, v);
1071 }
1072
1073 static __always_inline void
1074 atomic64_xor(s64 i, atomic64_t *v)
1075 {
1076         instrument_atomic_read_write(v, sizeof(*v));
1077         arch_atomic64_xor(i, v);
1078 }
1079
1080 static __always_inline s64
1081 atomic64_fetch_xor(s64 i, atomic64_t *v)
1082 {
1083         kcsan_mb();
1084         instrument_atomic_read_write(v, sizeof(*v));
1085         return arch_atomic64_fetch_xor(i, v);
1086 }
1087
1088 static __always_inline s64
1089 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1090 {
1091         instrument_atomic_read_write(v, sizeof(*v));
1092         return arch_atomic64_fetch_xor_acquire(i, v);
1093 }
1094
1095 static __always_inline s64
1096 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1097 {
1098         kcsan_release();
1099         instrument_atomic_read_write(v, sizeof(*v));
1100         return arch_atomic64_fetch_xor_release(i, v);
1101 }
1102
1103 static __always_inline s64
1104 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1105 {
1106         instrument_atomic_read_write(v, sizeof(*v));
1107         return arch_atomic64_fetch_xor_relaxed(i, v);
1108 }
1109
1110 static __always_inline s64
1111 atomic64_xchg(atomic64_t *v, s64 i)
1112 {
1113         kcsan_mb();
1114         instrument_atomic_read_write(v, sizeof(*v));
1115         return arch_atomic64_xchg(v, i);
1116 }
1117
1118 static __always_inline s64
1119 atomic64_xchg_acquire(atomic64_t *v, s64 i)
1120 {
1121         instrument_atomic_read_write(v, sizeof(*v));
1122         return arch_atomic64_xchg_acquire(v, i);
1123 }
1124
1125 static __always_inline s64
1126 atomic64_xchg_release(atomic64_t *v, s64 i)
1127 {
1128         kcsan_release();
1129         instrument_atomic_read_write(v, sizeof(*v));
1130         return arch_atomic64_xchg_release(v, i);
1131 }
1132
1133 static __always_inline s64
1134 atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1135 {
1136         instrument_atomic_read_write(v, sizeof(*v));
1137         return arch_atomic64_xchg_relaxed(v, i);
1138 }
1139
1140 static __always_inline s64
1141 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1142 {
1143         kcsan_mb();
1144         instrument_atomic_read_write(v, sizeof(*v));
1145         return arch_atomic64_cmpxchg(v, old, new);
1146 }
1147
1148 static __always_inline s64
1149 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1150 {
1151         instrument_atomic_read_write(v, sizeof(*v));
1152         return arch_atomic64_cmpxchg_acquire(v, old, new);
1153 }
1154
1155 static __always_inline s64
1156 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1157 {
1158         kcsan_release();
1159         instrument_atomic_read_write(v, sizeof(*v));
1160         return arch_atomic64_cmpxchg_release(v, old, new);
1161 }
1162
1163 static __always_inline s64
1164 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1165 {
1166         instrument_atomic_read_write(v, sizeof(*v));
1167         return arch_atomic64_cmpxchg_relaxed(v, old, new);
1168 }
1169
1170 static __always_inline bool
1171 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1172 {
1173         kcsan_mb();
1174         instrument_atomic_read_write(v, sizeof(*v));
1175         instrument_atomic_read_write(old, sizeof(*old));
1176         return arch_atomic64_try_cmpxchg(v, old, new);
1177 }
1178
1179 static __always_inline bool
1180 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1181 {
1182         instrument_atomic_read_write(v, sizeof(*v));
1183         instrument_atomic_read_write(old, sizeof(*old));
1184         return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1185 }
1186
1187 static __always_inline bool
1188 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1189 {
1190         kcsan_release();
1191         instrument_atomic_read_write(v, sizeof(*v));
1192         instrument_atomic_read_write(old, sizeof(*old));
1193         return arch_atomic64_try_cmpxchg_release(v, old, new);
1194 }
1195
1196 static __always_inline bool
1197 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1198 {
1199         instrument_atomic_read_write(v, sizeof(*v));
1200         instrument_atomic_read_write(old, sizeof(*old));
1201         return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1202 }
1203
1204 static __always_inline bool
1205 atomic64_sub_and_test(s64 i, atomic64_t *v)
1206 {
1207         kcsan_mb();
1208         instrument_atomic_read_write(v, sizeof(*v));
1209         return arch_atomic64_sub_and_test(i, v);
1210 }
1211
1212 static __always_inline bool
1213 atomic64_dec_and_test(atomic64_t *v)
1214 {
1215         kcsan_mb();
1216         instrument_atomic_read_write(v, sizeof(*v));
1217         return arch_atomic64_dec_and_test(v);
1218 }
1219
1220 static __always_inline bool
1221 atomic64_inc_and_test(atomic64_t *v)
1222 {
1223         kcsan_mb();
1224         instrument_atomic_read_write(v, sizeof(*v));
1225         return arch_atomic64_inc_and_test(v);
1226 }
1227
1228 static __always_inline bool
1229 atomic64_add_negative(s64 i, atomic64_t *v)
1230 {
1231         kcsan_mb();
1232         instrument_atomic_read_write(v, sizeof(*v));
1233         return arch_atomic64_add_negative(i, v);
1234 }
1235
1236 static __always_inline bool
1237 atomic64_add_negative_acquire(s64 i, atomic64_t *v)
1238 {
1239         instrument_atomic_read_write(v, sizeof(*v));
1240         return arch_atomic64_add_negative_acquire(i, v);
1241 }
1242
1243 static __always_inline bool
1244 atomic64_add_negative_release(s64 i, atomic64_t *v)
1245 {
1246         kcsan_release();
1247         instrument_atomic_read_write(v, sizeof(*v));
1248         return arch_atomic64_add_negative_release(i, v);
1249 }
1250
1251 static __always_inline bool
1252 atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
1253 {
1254         instrument_atomic_read_write(v, sizeof(*v));
1255         return arch_atomic64_add_negative_relaxed(i, v);
1256 }
1257
1258 static __always_inline s64
1259 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1260 {
1261         kcsan_mb();
1262         instrument_atomic_read_write(v, sizeof(*v));
1263         return arch_atomic64_fetch_add_unless(v, a, u);
1264 }
1265
1266 static __always_inline bool
1267 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1268 {
1269         kcsan_mb();
1270         instrument_atomic_read_write(v, sizeof(*v));
1271         return arch_atomic64_add_unless(v, a, u);
1272 }
1273
1274 static __always_inline bool
1275 atomic64_inc_not_zero(atomic64_t *v)
1276 {
1277         kcsan_mb();
1278         instrument_atomic_read_write(v, sizeof(*v));
1279         return arch_atomic64_inc_not_zero(v);
1280 }
1281
1282 static __always_inline bool
1283 atomic64_inc_unless_negative(atomic64_t *v)
1284 {
1285         kcsan_mb();
1286         instrument_atomic_read_write(v, sizeof(*v));
1287         return arch_atomic64_inc_unless_negative(v);
1288 }
1289
1290 static __always_inline bool
1291 atomic64_dec_unless_positive(atomic64_t *v)
1292 {
1293         kcsan_mb();
1294         instrument_atomic_read_write(v, sizeof(*v));
1295         return arch_atomic64_dec_unless_positive(v);
1296 }
1297
1298 static __always_inline s64
1299 atomic64_dec_if_positive(atomic64_t *v)
1300 {
1301         kcsan_mb();
1302         instrument_atomic_read_write(v, sizeof(*v));
1303         return arch_atomic64_dec_if_positive(v);
1304 }
1305
1306 static __always_inline long
1307 atomic_long_read(const atomic_long_t *v)
1308 {
1309         instrument_atomic_read(v, sizeof(*v));
1310         return arch_atomic_long_read(v);
1311 }
1312
1313 static __always_inline long
1314 atomic_long_read_acquire(const atomic_long_t *v)
1315 {
1316         instrument_atomic_read(v, sizeof(*v));
1317         return arch_atomic_long_read_acquire(v);
1318 }
1319
1320 static __always_inline void
1321 atomic_long_set(atomic_long_t *v, long i)
1322 {
1323         instrument_atomic_write(v, sizeof(*v));
1324         arch_atomic_long_set(v, i);
1325 }
1326
1327 static __always_inline void
1328 atomic_long_set_release(atomic_long_t *v, long i)
1329 {
1330         kcsan_release();
1331         instrument_atomic_write(v, sizeof(*v));
1332         arch_atomic_long_set_release(v, i);
1333 }
1334
1335 static __always_inline void
1336 atomic_long_add(long i, atomic_long_t *v)
1337 {
1338         instrument_atomic_read_write(v, sizeof(*v));
1339         arch_atomic_long_add(i, v);
1340 }
1341
1342 static __always_inline long
1343 atomic_long_add_return(long i, atomic_long_t *v)
1344 {
1345         kcsan_mb();
1346         instrument_atomic_read_write(v, sizeof(*v));
1347         return arch_atomic_long_add_return(i, v);
1348 }
1349
1350 static __always_inline long
1351 atomic_long_add_return_acquire(long i, atomic_long_t *v)
1352 {
1353         instrument_atomic_read_write(v, sizeof(*v));
1354         return arch_atomic_long_add_return_acquire(i, v);
1355 }
1356
1357 static __always_inline long
1358 atomic_long_add_return_release(long i, atomic_long_t *v)
1359 {
1360         kcsan_release();
1361         instrument_atomic_read_write(v, sizeof(*v));
1362         return arch_atomic_long_add_return_release(i, v);
1363 }
1364
1365 static __always_inline long
1366 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1367 {
1368         instrument_atomic_read_write(v, sizeof(*v));
1369         return arch_atomic_long_add_return_relaxed(i, v);
1370 }
1371
1372 static __always_inline long
1373 atomic_long_fetch_add(long i, atomic_long_t *v)
1374 {
1375         kcsan_mb();
1376         instrument_atomic_read_write(v, sizeof(*v));
1377         return arch_atomic_long_fetch_add(i, v);
1378 }
1379
1380 static __always_inline long
1381 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1382 {
1383         instrument_atomic_read_write(v, sizeof(*v));
1384         return arch_atomic_long_fetch_add_acquire(i, v);
1385 }
1386
1387 static __always_inline long
1388 atomic_long_fetch_add_release(long i, atomic_long_t *v)
1389 {
1390         kcsan_release();
1391         instrument_atomic_read_write(v, sizeof(*v));
1392         return arch_atomic_long_fetch_add_release(i, v);
1393 }
1394
1395 static __always_inline long
1396 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1397 {
1398         instrument_atomic_read_write(v, sizeof(*v));
1399         return arch_atomic_long_fetch_add_relaxed(i, v);
1400 }
1401
1402 static __always_inline void
1403 atomic_long_sub(long i, atomic_long_t *v)
1404 {
1405         instrument_atomic_read_write(v, sizeof(*v));
1406         arch_atomic_long_sub(i, v);
1407 }
1408
1409 static __always_inline long
1410 atomic_long_sub_return(long i, atomic_long_t *v)
1411 {
1412         kcsan_mb();
1413         instrument_atomic_read_write(v, sizeof(*v));
1414         return arch_atomic_long_sub_return(i, v);
1415 }
1416
1417 static __always_inline long
1418 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1419 {
1420         instrument_atomic_read_write(v, sizeof(*v));
1421         return arch_atomic_long_sub_return_acquire(i, v);
1422 }
1423
1424 static __always_inline long
1425 atomic_long_sub_return_release(long i, atomic_long_t *v)
1426 {
1427         kcsan_release();
1428         instrument_atomic_read_write(v, sizeof(*v));
1429         return arch_atomic_long_sub_return_release(i, v);
1430 }
1431
1432 static __always_inline long
1433 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1434 {
1435         instrument_atomic_read_write(v, sizeof(*v));
1436         return arch_atomic_long_sub_return_relaxed(i, v);
1437 }
1438
1439 static __always_inline long
1440 atomic_long_fetch_sub(long i, atomic_long_t *v)
1441 {
1442         kcsan_mb();
1443         instrument_atomic_read_write(v, sizeof(*v));
1444         return arch_atomic_long_fetch_sub(i, v);
1445 }
1446
1447 static __always_inline long
1448 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1449 {
1450         instrument_atomic_read_write(v, sizeof(*v));
1451         return arch_atomic_long_fetch_sub_acquire(i, v);
1452 }
1453
1454 static __always_inline long
1455 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1456 {
1457         kcsan_release();
1458         instrument_atomic_read_write(v, sizeof(*v));
1459         return arch_atomic_long_fetch_sub_release(i, v);
1460 }
1461
1462 static __always_inline long
1463 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1464 {
1465         instrument_atomic_read_write(v, sizeof(*v));
1466         return arch_atomic_long_fetch_sub_relaxed(i, v);
1467 }
1468
1469 static __always_inline void
1470 atomic_long_inc(atomic_long_t *v)
1471 {
1472         instrument_atomic_read_write(v, sizeof(*v));
1473         arch_atomic_long_inc(v);
1474 }
1475
1476 static __always_inline long
1477 atomic_long_inc_return(atomic_long_t *v)
1478 {
1479         kcsan_mb();
1480         instrument_atomic_read_write(v, sizeof(*v));
1481         return arch_atomic_long_inc_return(v);
1482 }
1483
1484 static __always_inline long
1485 atomic_long_inc_return_acquire(atomic_long_t *v)
1486 {
1487         instrument_atomic_read_write(v, sizeof(*v));
1488         return arch_atomic_long_inc_return_acquire(v);
1489 }
1490
1491 static __always_inline long
1492 atomic_long_inc_return_release(atomic_long_t *v)
1493 {
1494         kcsan_release();
1495         instrument_atomic_read_write(v, sizeof(*v));
1496         return arch_atomic_long_inc_return_release(v);
1497 }
1498
1499 static __always_inline long
1500 atomic_long_inc_return_relaxed(atomic_long_t *v)
1501 {
1502         instrument_atomic_read_write(v, sizeof(*v));
1503         return arch_atomic_long_inc_return_relaxed(v);
1504 }
1505
1506 static __always_inline long
1507 atomic_long_fetch_inc(atomic_long_t *v)
1508 {
1509         kcsan_mb();
1510         instrument_atomic_read_write(v, sizeof(*v));
1511         return arch_atomic_long_fetch_inc(v);
1512 }
1513
1514 static __always_inline long
1515 atomic_long_fetch_inc_acquire(atomic_long_t *v)
1516 {
1517         instrument_atomic_read_write(v, sizeof(*v));
1518         return arch_atomic_long_fetch_inc_acquire(v);
1519 }
1520
1521 static __always_inline long
1522 atomic_long_fetch_inc_release(atomic_long_t *v)
1523 {
1524         kcsan_release();
1525         instrument_atomic_read_write(v, sizeof(*v));
1526         return arch_atomic_long_fetch_inc_release(v);
1527 }
1528
1529 static __always_inline long
1530 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1531 {
1532         instrument_atomic_read_write(v, sizeof(*v));
1533         return arch_atomic_long_fetch_inc_relaxed(v);
1534 }
1535
1536 static __always_inline void
1537 atomic_long_dec(atomic_long_t *v)
1538 {
1539         instrument_atomic_read_write(v, sizeof(*v));
1540         arch_atomic_long_dec(v);
1541 }
1542
1543 static __always_inline long
1544 atomic_long_dec_return(atomic_long_t *v)
1545 {
1546         kcsan_mb();
1547         instrument_atomic_read_write(v, sizeof(*v));
1548         return arch_atomic_long_dec_return(v);
1549 }
1550
1551 static __always_inline long
1552 atomic_long_dec_return_acquire(atomic_long_t *v)
1553 {
1554         instrument_atomic_read_write(v, sizeof(*v));
1555         return arch_atomic_long_dec_return_acquire(v);
1556 }
1557
1558 static __always_inline long
1559 atomic_long_dec_return_release(atomic_long_t *v)
1560 {
1561         kcsan_release();
1562         instrument_atomic_read_write(v, sizeof(*v));
1563         return arch_atomic_long_dec_return_release(v);
1564 }
1565
1566 static __always_inline long
1567 atomic_long_dec_return_relaxed(atomic_long_t *v)
1568 {
1569         instrument_atomic_read_write(v, sizeof(*v));
1570         return arch_atomic_long_dec_return_relaxed(v);
1571 }
1572
1573 static __always_inline long
1574 atomic_long_fetch_dec(atomic_long_t *v)
1575 {
1576         kcsan_mb();
1577         instrument_atomic_read_write(v, sizeof(*v));
1578         return arch_atomic_long_fetch_dec(v);
1579 }
1580
1581 static __always_inline long
1582 atomic_long_fetch_dec_acquire(atomic_long_t *v)
1583 {
1584         instrument_atomic_read_write(v, sizeof(*v));
1585         return arch_atomic_long_fetch_dec_acquire(v);
1586 }
1587
1588 static __always_inline long
1589 atomic_long_fetch_dec_release(atomic_long_t *v)
1590 {
1591         kcsan_release();
1592         instrument_atomic_read_write(v, sizeof(*v));
1593         return arch_atomic_long_fetch_dec_release(v);
1594 }
1595
1596 static __always_inline long
1597 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1598 {
1599         instrument_atomic_read_write(v, sizeof(*v));
1600         return arch_atomic_long_fetch_dec_relaxed(v);
1601 }
1602
1603 static __always_inline void
1604 atomic_long_and(long i, atomic_long_t *v)
1605 {
1606         instrument_atomic_read_write(v, sizeof(*v));
1607         arch_atomic_long_and(i, v);
1608 }
1609
1610 static __always_inline long
1611 atomic_long_fetch_and(long i, atomic_long_t *v)
1612 {
1613         kcsan_mb();
1614         instrument_atomic_read_write(v, sizeof(*v));
1615         return arch_atomic_long_fetch_and(i, v);
1616 }
1617
1618 static __always_inline long
1619 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1620 {
1621         instrument_atomic_read_write(v, sizeof(*v));
1622         return arch_atomic_long_fetch_and_acquire(i, v);
1623 }
1624
1625 static __always_inline long
1626 atomic_long_fetch_and_release(long i, atomic_long_t *v)
1627 {
1628         kcsan_release();
1629         instrument_atomic_read_write(v, sizeof(*v));
1630         return arch_atomic_long_fetch_and_release(i, v);
1631 }
1632
1633 static __always_inline long
1634 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1635 {
1636         instrument_atomic_read_write(v, sizeof(*v));
1637         return arch_atomic_long_fetch_and_relaxed(i, v);
1638 }
1639
1640 static __always_inline void
1641 atomic_long_andnot(long i, atomic_long_t *v)
1642 {
1643         instrument_atomic_read_write(v, sizeof(*v));
1644         arch_atomic_long_andnot(i, v);
1645 }
1646
1647 static __always_inline long
1648 atomic_long_fetch_andnot(long i, atomic_long_t *v)
1649 {
1650         kcsan_mb();
1651         instrument_atomic_read_write(v, sizeof(*v));
1652         return arch_atomic_long_fetch_andnot(i, v);
1653 }
1654
1655 static __always_inline long
1656 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1657 {
1658         instrument_atomic_read_write(v, sizeof(*v));
1659         return arch_atomic_long_fetch_andnot_acquire(i, v);
1660 }
1661
1662 static __always_inline long
1663 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1664 {
1665         kcsan_release();
1666         instrument_atomic_read_write(v, sizeof(*v));
1667         return arch_atomic_long_fetch_andnot_release(i, v);
1668 }
1669
1670 static __always_inline long
1671 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1672 {
1673         instrument_atomic_read_write(v, sizeof(*v));
1674         return arch_atomic_long_fetch_andnot_relaxed(i, v);
1675 }
1676
1677 static __always_inline void
1678 atomic_long_or(long i, atomic_long_t *v)
1679 {
1680         instrument_atomic_read_write(v, sizeof(*v));
1681         arch_atomic_long_or(i, v);
1682 }
1683
1684 static __always_inline long
1685 atomic_long_fetch_or(long i, atomic_long_t *v)
1686 {
1687         kcsan_mb();
1688         instrument_atomic_read_write(v, sizeof(*v));
1689         return arch_atomic_long_fetch_or(i, v);
1690 }
1691
1692 static __always_inline long
1693 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1694 {
1695         instrument_atomic_read_write(v, sizeof(*v));
1696         return arch_atomic_long_fetch_or_acquire(i, v);
1697 }
1698
1699 static __always_inline long
1700 atomic_long_fetch_or_release(long i, atomic_long_t *v)
1701 {
1702         kcsan_release();
1703         instrument_atomic_read_write(v, sizeof(*v));
1704         return arch_atomic_long_fetch_or_release(i, v);
1705 }
1706
1707 static __always_inline long
1708 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1709 {
1710         instrument_atomic_read_write(v, sizeof(*v));
1711         return arch_atomic_long_fetch_or_relaxed(i, v);
1712 }
1713
1714 static __always_inline void
1715 atomic_long_xor(long i, atomic_long_t *v)
1716 {
1717         instrument_atomic_read_write(v, sizeof(*v));
1718         arch_atomic_long_xor(i, v);
1719 }
1720
1721 static __always_inline long
1722 atomic_long_fetch_xor(long i, atomic_long_t *v)
1723 {
1724         kcsan_mb();
1725         instrument_atomic_read_write(v, sizeof(*v));
1726         return arch_atomic_long_fetch_xor(i, v);
1727 }
1728
1729 static __always_inline long
1730 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1731 {
1732         instrument_atomic_read_write(v, sizeof(*v));
1733         return arch_atomic_long_fetch_xor_acquire(i, v);
1734 }
1735
1736 static __always_inline long
1737 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1738 {
1739         kcsan_release();
1740         instrument_atomic_read_write(v, sizeof(*v));
1741         return arch_atomic_long_fetch_xor_release(i, v);
1742 }
1743
1744 static __always_inline long
1745 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1746 {
1747         instrument_atomic_read_write(v, sizeof(*v));
1748         return arch_atomic_long_fetch_xor_relaxed(i, v);
1749 }
1750
1751 static __always_inline long
1752 atomic_long_xchg(atomic_long_t *v, long i)
1753 {
1754         kcsan_mb();
1755         instrument_atomic_read_write(v, sizeof(*v));
1756         return arch_atomic_long_xchg(v, i);
1757 }
1758
1759 static __always_inline long
1760 atomic_long_xchg_acquire(atomic_long_t *v, long i)
1761 {
1762         instrument_atomic_read_write(v, sizeof(*v));
1763         return arch_atomic_long_xchg_acquire(v, i);
1764 }
1765
1766 static __always_inline long
1767 atomic_long_xchg_release(atomic_long_t *v, long i)
1768 {
1769         kcsan_release();
1770         instrument_atomic_read_write(v, sizeof(*v));
1771         return arch_atomic_long_xchg_release(v, i);
1772 }
1773
1774 static __always_inline long
1775 atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1776 {
1777         instrument_atomic_read_write(v, sizeof(*v));
1778         return arch_atomic_long_xchg_relaxed(v, i);
1779 }
1780
1781 static __always_inline long
1782 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1783 {
1784         kcsan_mb();
1785         instrument_atomic_read_write(v, sizeof(*v));
1786         return arch_atomic_long_cmpxchg(v, old, new);
1787 }
1788
1789 static __always_inline long
1790 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1791 {
1792         instrument_atomic_read_write(v, sizeof(*v));
1793         return arch_atomic_long_cmpxchg_acquire(v, old, new);
1794 }
1795
1796 static __always_inline long
1797 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1798 {
1799         kcsan_release();
1800         instrument_atomic_read_write(v, sizeof(*v));
1801         return arch_atomic_long_cmpxchg_release(v, old, new);
1802 }
1803
1804 static __always_inline long
1805 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1806 {
1807         instrument_atomic_read_write(v, sizeof(*v));
1808         return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1809 }
1810
1811 static __always_inline bool
1812 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1813 {
1814         kcsan_mb();
1815         instrument_atomic_read_write(v, sizeof(*v));
1816         instrument_atomic_read_write(old, sizeof(*old));
1817         return arch_atomic_long_try_cmpxchg(v, old, new);
1818 }
1819
1820 static __always_inline bool
1821 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1822 {
1823         instrument_atomic_read_write(v, sizeof(*v));
1824         instrument_atomic_read_write(old, sizeof(*old));
1825         return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1826 }
1827
1828 static __always_inline bool
1829 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1830 {
1831         kcsan_release();
1832         instrument_atomic_read_write(v, sizeof(*v));
1833         instrument_atomic_read_write(old, sizeof(*old));
1834         return arch_atomic_long_try_cmpxchg_release(v, old, new);
1835 }
1836
1837 static __always_inline bool
1838 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1839 {
1840         instrument_atomic_read_write(v, sizeof(*v));
1841         instrument_atomic_read_write(old, sizeof(*old));
1842         return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1843 }
1844
1845 static __always_inline bool
1846 atomic_long_sub_and_test(long i, atomic_long_t *v)
1847 {
1848         kcsan_mb();
1849         instrument_atomic_read_write(v, sizeof(*v));
1850         return arch_atomic_long_sub_and_test(i, v);
1851 }
1852
1853 static __always_inline bool
1854 atomic_long_dec_and_test(atomic_long_t *v)
1855 {
1856         kcsan_mb();
1857         instrument_atomic_read_write(v, sizeof(*v));
1858         return arch_atomic_long_dec_and_test(v);
1859 }
1860
1861 static __always_inline bool
1862 atomic_long_inc_and_test(atomic_long_t *v)
1863 {
1864         kcsan_mb();
1865         instrument_atomic_read_write(v, sizeof(*v));
1866         return arch_atomic_long_inc_and_test(v);
1867 }
1868
1869 static __always_inline bool
1870 atomic_long_add_negative(long i, atomic_long_t *v)
1871 {
1872         kcsan_mb();
1873         instrument_atomic_read_write(v, sizeof(*v));
1874         return arch_atomic_long_add_negative(i, v);
1875 }
1876
1877 static __always_inline bool
1878 atomic_long_add_negative_acquire(long i, atomic_long_t *v)
1879 {
1880         instrument_atomic_read_write(v, sizeof(*v));
1881         return arch_atomic_long_add_negative_acquire(i, v);
1882 }
1883
1884 static __always_inline bool
1885 atomic_long_add_negative_release(long i, atomic_long_t *v)
1886 {
1887         kcsan_release();
1888         instrument_atomic_read_write(v, sizeof(*v));
1889         return arch_atomic_long_add_negative_release(i, v);
1890 }
1891
1892 static __always_inline bool
1893 atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
1894 {
1895         instrument_atomic_read_write(v, sizeof(*v));
1896         return arch_atomic_long_add_negative_relaxed(i, v);
1897 }
1898
1899 static __always_inline long
1900 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1901 {
1902         kcsan_mb();
1903         instrument_atomic_read_write(v, sizeof(*v));
1904         return arch_atomic_long_fetch_add_unless(v, a, u);
1905 }
1906
1907 static __always_inline bool
1908 atomic_long_add_unless(atomic_long_t *v, long a, long u)
1909 {
1910         kcsan_mb();
1911         instrument_atomic_read_write(v, sizeof(*v));
1912         return arch_atomic_long_add_unless(v, a, u);
1913 }
1914
1915 static __always_inline bool
1916 atomic_long_inc_not_zero(atomic_long_t *v)
1917 {
1918         kcsan_mb();
1919         instrument_atomic_read_write(v, sizeof(*v));
1920         return arch_atomic_long_inc_not_zero(v);
1921 }
1922
1923 static __always_inline bool
1924 atomic_long_inc_unless_negative(atomic_long_t *v)
1925 {
1926         kcsan_mb();
1927         instrument_atomic_read_write(v, sizeof(*v));
1928         return arch_atomic_long_inc_unless_negative(v);
1929 }
1930
1931 static __always_inline bool
1932 atomic_long_dec_unless_positive(atomic_long_t *v)
1933 {
1934         kcsan_mb();
1935         instrument_atomic_read_write(v, sizeof(*v));
1936         return arch_atomic_long_dec_unless_positive(v);
1937 }
1938
1939 static __always_inline long
1940 atomic_long_dec_if_positive(atomic_long_t *v)
1941 {
1942         kcsan_mb();
1943         instrument_atomic_read_write(v, sizeof(*v));
1944         return arch_atomic_long_dec_if_positive(v);
1945 }
1946
1947 #define xchg(ptr, ...) \
1948 ({ \
1949         typeof(ptr) __ai_ptr = (ptr); \
1950         kcsan_mb(); \
1951         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1952         arch_xchg(__ai_ptr, __VA_ARGS__); \
1953 })
1954
1955 #define xchg_acquire(ptr, ...) \
1956 ({ \
1957         typeof(ptr) __ai_ptr = (ptr); \
1958         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1959         arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1960 })
1961
1962 #define xchg_release(ptr, ...) \
1963 ({ \
1964         typeof(ptr) __ai_ptr = (ptr); \
1965         kcsan_release(); \
1966         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1967         arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1968 })
1969
1970 #define xchg_relaxed(ptr, ...) \
1971 ({ \
1972         typeof(ptr) __ai_ptr = (ptr); \
1973         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1974         arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1975 })
1976
1977 #define cmpxchg(ptr, ...) \
1978 ({ \
1979         typeof(ptr) __ai_ptr = (ptr); \
1980         kcsan_mb(); \
1981         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1982         arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1983 })
1984
1985 #define cmpxchg_acquire(ptr, ...) \
1986 ({ \
1987         typeof(ptr) __ai_ptr = (ptr); \
1988         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1989         arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1990 })
1991
1992 #define cmpxchg_release(ptr, ...) \
1993 ({ \
1994         typeof(ptr) __ai_ptr = (ptr); \
1995         kcsan_release(); \
1996         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1997         arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1998 })
1999
2000 #define cmpxchg_relaxed(ptr, ...) \
2001 ({ \
2002         typeof(ptr) __ai_ptr = (ptr); \
2003         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2004         arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
2005 })
2006
2007 #define cmpxchg64(ptr, ...) \
2008 ({ \
2009         typeof(ptr) __ai_ptr = (ptr); \
2010         kcsan_mb(); \
2011         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2012         arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
2013 })
2014
2015 #define cmpxchg64_acquire(ptr, ...) \
2016 ({ \
2017         typeof(ptr) __ai_ptr = (ptr); \
2018         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2019         arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
2020 })
2021
2022 #define cmpxchg64_release(ptr, ...) \
2023 ({ \
2024         typeof(ptr) __ai_ptr = (ptr); \
2025         kcsan_release(); \
2026         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2027         arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
2028 })
2029
2030 #define cmpxchg64_relaxed(ptr, ...) \
2031 ({ \
2032         typeof(ptr) __ai_ptr = (ptr); \
2033         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034         arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
2035 })
2036
2037 #define try_cmpxchg(ptr, oldp, ...) \
2038 ({ \
2039         typeof(ptr) __ai_ptr = (ptr); \
2040         typeof(oldp) __ai_oldp = (oldp); \
2041         kcsan_mb(); \
2042         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043         instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044         arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045 })
2046
2047 #define try_cmpxchg_acquire(ptr, oldp, ...) \
2048 ({ \
2049         typeof(ptr) __ai_ptr = (ptr); \
2050         typeof(oldp) __ai_oldp = (oldp); \
2051         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2052         instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2053         arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2054 })
2055
2056 #define try_cmpxchg_release(ptr, oldp, ...) \
2057 ({ \
2058         typeof(ptr) __ai_ptr = (ptr); \
2059         typeof(oldp) __ai_oldp = (oldp); \
2060         kcsan_release(); \
2061         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2062         instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2063         arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2064 })
2065
2066 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
2067 ({ \
2068         typeof(ptr) __ai_ptr = (ptr); \
2069         typeof(oldp) __ai_oldp = (oldp); \
2070         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2071         instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2072         arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2073 })
2074
2075 #define try_cmpxchg64(ptr, oldp, ...) \
2076 ({ \
2077         typeof(ptr) __ai_ptr = (ptr); \
2078         typeof(oldp) __ai_oldp = (oldp); \
2079         kcsan_mb(); \
2080         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2081         instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2082         arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2083 })
2084
2085 #define try_cmpxchg64_acquire(ptr, oldp, ...) \
2086 ({ \
2087         typeof(ptr) __ai_ptr = (ptr); \
2088         typeof(oldp) __ai_oldp = (oldp); \
2089         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2090         instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2091         arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2092 })
2093
2094 #define try_cmpxchg64_release(ptr, oldp, ...) \
2095 ({ \
2096         typeof(ptr) __ai_ptr = (ptr); \
2097         typeof(oldp) __ai_oldp = (oldp); \
2098         kcsan_release(); \
2099         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2100         instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2101         arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2102 })
2103
2104 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
2105 ({ \
2106         typeof(ptr) __ai_ptr = (ptr); \
2107         typeof(oldp) __ai_oldp = (oldp); \
2108         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2109         instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2110         arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2111 })
2112
2113 #define cmpxchg_local(ptr, ...) \
2114 ({ \
2115         typeof(ptr) __ai_ptr = (ptr); \
2116         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2117         arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2118 })
2119
2120 #define cmpxchg64_local(ptr, ...) \
2121 ({ \
2122         typeof(ptr) __ai_ptr = (ptr); \
2123         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2124         arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2125 })
2126
2127 #define sync_cmpxchg(ptr, ...) \
2128 ({ \
2129         typeof(ptr) __ai_ptr = (ptr); \
2130         kcsan_mb(); \
2131         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2132         arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2133 })
2134
2135 #define try_cmpxchg_local(ptr, oldp, ...) \
2136 ({ \
2137         typeof(ptr) __ai_ptr = (ptr); \
2138         typeof(oldp) __ai_oldp = (oldp); \
2139         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2140         instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2141         arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2142 })
2143
2144 #define try_cmpxchg64_local(ptr, oldp, ...) \
2145 ({ \
2146         typeof(ptr) __ai_ptr = (ptr); \
2147         typeof(oldp) __ai_oldp = (oldp); \
2148         instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2149         instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2150         arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2151 })
2152
2153 #define cmpxchg_double(ptr, ...) \
2154 ({ \
2155         typeof(ptr) __ai_ptr = (ptr); \
2156         kcsan_mb(); \
2157         instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2158         arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2159 })
2160
2161
2162 #define cmpxchg_double_local(ptr, ...) \
2163 ({ \
2164         typeof(ptr) __ai_ptr = (ptr); \
2165         instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2166         arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2167 })
2168
2169 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2170 // 6b513a42e1a1b5962532a019b7fc91eaa044ad5e