include/linux/atomic/atomic-instrumented.h

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
   4 // DO NOT MODIFY THIS FILE DIRECTLY
   5
   6 /*
   7  * This file provides wrappers with KASAN instrumentation for atomic operations.
   8  * To use this functionality an arch's atomic.h file needs to define all
   9  * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
  10  * this file at the end. This file provides atomic_read() that forwards to
  11  * arch_atomic_read() for actual atomic operation.
  12  * Note: if an arch atomic operation is implemented by means of other atomic
  13  * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
  14  * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
  15  * double instrumentation.
  16  */
  17 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
  18 #define _LINUX_ATOMIC_INSTRUMENTED_H
  19
  20 #include <linux/build_bug.h>
  21 #include <linux/compiler.h>
  22 #include <linux/instrumented.h>
  23
  24 static __always_inline int
  25 atomic_read(const atomic_t *v)
  26 {
  27         instrument_atomic_read(v, sizeof(*v));
  28         return arch_atomic_read(v);
  29 }
  30
  31 static __always_inline int
  32 atomic_read_acquire(const atomic_t *v)
  33 {
  34         instrument_atomic_read(v, sizeof(*v));
  35         return arch_atomic_read_acquire(v);
  36 }
  37
  38 static __always_inline void
  39 atomic_set(atomic_t *v, int i)
  40 {
  41         instrument_atomic_write(v, sizeof(*v));
  42         arch_atomic_set(v, i);
  43 }
  44
  45 static __always_inline void
  46 atomic_set_release(atomic_t *v, int i)
  47 {
  48         kcsan_release();
  49         instrument_atomic_write(v, sizeof(*v));
  50         arch_atomic_set_release(v, i);
  51 }
  52
  53 static __always_inline void
  54 atomic_add(int i, atomic_t *v)
  55 {
  56         instrument_atomic_read_write(v, sizeof(*v));
  57         arch_atomic_add(i, v);
  58 }
  59
  60 static __always_inline int
  61 atomic_add_return(int i, atomic_t *v)
  62 {
  63         kcsan_mb();
  64         instrument_atomic_read_write(v, sizeof(*v));
  65         return arch_atomic_add_return(i, v);
  66 }
  67
  68 static __always_inline int
  69 atomic_add_return_acquire(int i, atomic_t *v)
  70 {
  71         instrument_atomic_read_write(v, sizeof(*v));
  72         return arch_atomic_add_return_acquire(i, v);
  73 }
  74
  75 static __always_inline int
  76 atomic_add_return_release(int i, atomic_t *v)
  77 {
  78         kcsan_release();
  79         instrument_atomic_read_write(v, sizeof(*v));
  80         return arch_atomic_add_return_release(i, v);
  81 }
  82
  83 static __always_inline int
  84 atomic_add_return_relaxed(int i, atomic_t *v)
  85 {
  86         instrument_atomic_read_write(v, sizeof(*v));
  87         return arch_atomic_add_return_relaxed(i, v);
  88 }
  89
  90 static __always_inline int
  91 atomic_fetch_add(int i, atomic_t *v)
  92 {
  93         kcsan_mb();
  94         instrument_atomic_read_write(v, sizeof(*v));
  95         return arch_atomic_fetch_add(i, v);
  96 }
  97
  98 static __always_inline int
  99 atomic_fetch_add_acquire(int i, atomic_t *v)
 100 {
 101         instrument_atomic_read_write(v, sizeof(*v));
 102         return arch_atomic_fetch_add_acquire(i, v);
 103 }
 104
 105 static __always_inline int
 106 atomic_fetch_add_release(int i, atomic_t *v)
 107 {
 108         kcsan_release();
 109         instrument_atomic_read_write(v, sizeof(*v));
 110         return arch_atomic_fetch_add_release(i, v);
 111 }
 112
 113 static __always_inline int
 114 atomic_fetch_add_relaxed(int i, atomic_t *v)
 115 {
 116         instrument_atomic_read_write(v, sizeof(*v));
 117         return arch_atomic_fetch_add_relaxed(i, v);
 118 }
 119
 120 static __always_inline void
 121 atomic_sub(int i, atomic_t *v)
 122 {
 123         instrument_atomic_read_write(v, sizeof(*v));
 124         arch_atomic_sub(i, v);
 125 }
 126
 127 static __always_inline int
 128 atomic_sub_return(int i, atomic_t *v)
 129 {
 130         kcsan_mb();
 131         instrument_atomic_read_write(v, sizeof(*v));
 132         return arch_atomic_sub_return(i, v);
 133 }
 134
 135 static __always_inline int
 136 atomic_sub_return_acquire(int i, atomic_t *v)
 137 {
 138         instrument_atomic_read_write(v, sizeof(*v));
 139         return arch_atomic_sub_return_acquire(i, v);
 140 }
 141
 142 static __always_inline int
 143 atomic_sub_return_release(int i, atomic_t *v)
 144 {
 145         kcsan_release();
 146         instrument_atomic_read_write(v, sizeof(*v));
 147         return arch_atomic_sub_return_release(i, v);
 148 }
 149
 150 static __always_inline int
 151 atomic_sub_return_relaxed(int i, atomic_t *v)
 152 {
 153         instrument_atomic_read_write(v, sizeof(*v));
 154         return arch_atomic_sub_return_relaxed(i, v);
 155 }
 156
 157 static __always_inline int
 158 atomic_fetch_sub(int i, atomic_t *v)
 159 {
 160         kcsan_mb();
 161         instrument_atomic_read_write(v, sizeof(*v));
 162         return arch_atomic_fetch_sub(i, v);
 163 }
 164
 165 static __always_inline int
 166 atomic_fetch_sub_acquire(int i, atomic_t *v)
 167 {
 168         instrument_atomic_read_write(v, sizeof(*v));
 169         return arch_atomic_fetch_sub_acquire(i, v);
 170 }
 171
 172 static __always_inline int
 173 atomic_fetch_sub_release(int i, atomic_t *v)
 174 {
 175         kcsan_release();
 176         instrument_atomic_read_write(v, sizeof(*v));
 177         return arch_atomic_fetch_sub_release(i, v);
 178 }
 179
 180 static __always_inline int
 181 atomic_fetch_sub_relaxed(int i, atomic_t *v)
 182 {
 183         instrument_atomic_read_write(v, sizeof(*v));
 184         return arch_atomic_fetch_sub_relaxed(i, v);
 185 }
 186
 187 static __always_inline void
 188 atomic_inc(atomic_t *v)
 189 {
 190         instrument_atomic_read_write(v, sizeof(*v));
 191         arch_atomic_inc(v);
 192 }
 193
 194 static __always_inline int
 195 atomic_inc_return(atomic_t *v)
 196 {
 197         kcsan_mb();
 198         instrument_atomic_read_write(v, sizeof(*v));
 199         return arch_atomic_inc_return(v);
 200 }
 201
 202 static __always_inline int
 203 atomic_inc_return_acquire(atomic_t *v)
 204 {
 205         instrument_atomic_read_write(v, sizeof(*v));
 206         return arch_atomic_inc_return_acquire(v);
 207 }
 208
 209 static __always_inline int
 210 atomic_inc_return_release(atomic_t *v)
 211 {
 212         kcsan_release();
 213         instrument_atomic_read_write(v, sizeof(*v));
 214         return arch_atomic_inc_return_release(v);
 215 }
 216
 217 static __always_inline int
 218 atomic_inc_return_relaxed(atomic_t *v)
 219 {
 220         instrument_atomic_read_write(v, sizeof(*v));
 221         return arch_atomic_inc_return_relaxed(v);
 222 }
 223
 224 static __always_inline int
 225 atomic_fetch_inc(atomic_t *v)
 226 {
 227         kcsan_mb();
 228         instrument_atomic_read_write(v, sizeof(*v));
 229         return arch_atomic_fetch_inc(v);
 230 }
 231
 232 static __always_inline int
 233 atomic_fetch_inc_acquire(atomic_t *v)
 234 {
 235         instrument_atomic_read_write(v, sizeof(*v));
 236         return arch_atomic_fetch_inc_acquire(v);
 237 }
 238
 239 static __always_inline int
 240 atomic_fetch_inc_release(atomic_t *v)
 241 {
 242         kcsan_release();
 243         instrument_atomic_read_write(v, sizeof(*v));
 244         return arch_atomic_fetch_inc_release(v);
 245 }
 246
 247 static __always_inline int
 248 atomic_fetch_inc_relaxed(atomic_t *v)
 249 {
 250         instrument_atomic_read_write(v, sizeof(*v));
 251         return arch_atomic_fetch_inc_relaxed(v);
 252 }
 253
 254 static __always_inline void
 255 atomic_dec(atomic_t *v)
 256 {
 257         instrument_atomic_read_write(v, sizeof(*v));
 258         arch_atomic_dec(v);
 259 }
 260
 261 static __always_inline int
 262 atomic_dec_return(atomic_t *v)
 263 {
 264         kcsan_mb();
 265         instrument_atomic_read_write(v, sizeof(*v));
 266         return arch_atomic_dec_return(v);
 267 }
 268
 269 static __always_inline int
 270 atomic_dec_return_acquire(atomic_t *v)
 271 {
 272         instrument_atomic_read_write(v, sizeof(*v));
 273         return arch_atomic_dec_return_acquire(v);
 274 }
 275
 276 static __always_inline int
 277 atomic_dec_return_release(atomic_t *v)
 278 {
 279         kcsan_release();
 280         instrument_atomic_read_write(v, sizeof(*v));
 281         return arch_atomic_dec_return_release(v);
 282 }
 283
 284 static __always_inline int
 285 atomic_dec_return_relaxed(atomic_t *v)
 286 {
 287         instrument_atomic_read_write(v, sizeof(*v));
 288         return arch_atomic_dec_return_relaxed(v);
 289 }
 290
 291 static __always_inline int
 292 atomic_fetch_dec(atomic_t *v)
 293 {
 294         kcsan_mb();
 295         instrument_atomic_read_write(v, sizeof(*v));
 296         return arch_atomic_fetch_dec(v);
 297 }
 298
 299 static __always_inline int
 300 atomic_fetch_dec_acquire(atomic_t *v)
 301 {
 302         instrument_atomic_read_write(v, sizeof(*v));
 303         return arch_atomic_fetch_dec_acquire(v);
 304 }
 305
 306 static __always_inline int
 307 atomic_fetch_dec_release(atomic_t *v)
 308 {
 309         kcsan_release();
 310         instrument_atomic_read_write(v, sizeof(*v));
 311         return arch_atomic_fetch_dec_release(v);
 312 }
 313
 314 static __always_inline int
 315 atomic_fetch_dec_relaxed(atomic_t *v)
 316 {
 317         instrument_atomic_read_write(v, sizeof(*v));
 318         return arch_atomic_fetch_dec_relaxed(v);
 319 }
 320
 321 static __always_inline void
 322 atomic_and(int i, atomic_t *v)
 323 {
 324         instrument_atomic_read_write(v, sizeof(*v));
 325         arch_atomic_and(i, v);
 326 }
 327
 328 static __always_inline int
 329 atomic_fetch_and(int i, atomic_t *v)
 330 {
 331         kcsan_mb();
 332         instrument_atomic_read_write(v, sizeof(*v));
 333         return arch_atomic_fetch_and(i, v);
 334 }
 335
 336 static __always_inline int
 337 atomic_fetch_and_acquire(int i, atomic_t *v)
 338 {
 339         instrument_atomic_read_write(v, sizeof(*v));
 340         return arch_atomic_fetch_and_acquire(i, v);
 341 }
 342
 343 static __always_inline int
 344 atomic_fetch_and_release(int i, atomic_t *v)
 345 {
 346         kcsan_release();
 347         instrument_atomic_read_write(v, sizeof(*v));
 348         return arch_atomic_fetch_and_release(i, v);
 349 }
 350
 351 static __always_inline int
 352 atomic_fetch_and_relaxed(int i, atomic_t *v)
 353 {
 354         instrument_atomic_read_write(v, sizeof(*v));
 355         return arch_atomic_fetch_and_relaxed(i, v);
 356 }
 357
 358 static __always_inline void
 359 atomic_andnot(int i, atomic_t *v)
 360 {
 361         instrument_atomic_read_write(v, sizeof(*v));
 362         arch_atomic_andnot(i, v);
 363 }
 364
 365 static __always_inline int
 366 atomic_fetch_andnot(int i, atomic_t *v)
 367 {
 368         kcsan_mb();
 369         instrument_atomic_read_write(v, sizeof(*v));
 370         return arch_atomic_fetch_andnot(i, v);
 371 }
 372
 373 static __always_inline int
 374 atomic_fetch_andnot_acquire(int i, atomic_t *v)
 375 {
 376         instrument_atomic_read_write(v, sizeof(*v));
 377         return arch_atomic_fetch_andnot_acquire(i, v);
 378 }
 379
 380 static __always_inline int
 381 atomic_fetch_andnot_release(int i, atomic_t *v)
 382 {
 383         kcsan_release();
 384         instrument_atomic_read_write(v, sizeof(*v));
 385         return arch_atomic_fetch_andnot_release(i, v);
 386 }
 387
 388 static __always_inline int
 389 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
 390 {
 391         instrument_atomic_read_write(v, sizeof(*v));
 392         return arch_atomic_fetch_andnot_relaxed(i, v);
 393 }
 394
 395 static __always_inline void
 396 atomic_or(int i, atomic_t *v)
 397 {
 398         instrument_atomic_read_write(v, sizeof(*v));
 399         arch_atomic_or(i, v);
 400 }
 401
 402 static __always_inline int
 403 atomic_fetch_or(int i, atomic_t *v)
 404 {
 405         kcsan_mb();
 406         instrument_atomic_read_write(v, sizeof(*v));
 407         return arch_atomic_fetch_or(i, v);
 408 }
 409
 410 static __always_inline int
 411 atomic_fetch_or_acquire(int i, atomic_t *v)
 412 {
 413         instrument_atomic_read_write(v, sizeof(*v));
 414         return arch_atomic_fetch_or_acquire(i, v);
 415 }
 416
 417 static __always_inline int
 418 atomic_fetch_or_release(int i, atomic_t *v)
 419 {
 420         kcsan_release();
 421         instrument_atomic_read_write(v, sizeof(*v));
 422         return arch_atomic_fetch_or_release(i, v);
 423 }
 424
 425 static __always_inline int
 426 atomic_fetch_or_relaxed(int i, atomic_t *v)
 427 {
 428         instrument_atomic_read_write(v, sizeof(*v));
 429         return arch_atomic_fetch_or_relaxed(i, v);
 430 }
 431
 432 static __always_inline void
 433 atomic_xor(int i, atomic_t *v)
 434 {
 435         instrument_atomic_read_write(v, sizeof(*v));
 436         arch_atomic_xor(i, v);
 437 }
 438
 439 static __always_inline int
 440 atomic_fetch_xor(int i, atomic_t *v)
 441 {
 442         kcsan_mb();
 443         instrument_atomic_read_write(v, sizeof(*v));
 444         return arch_atomic_fetch_xor(i, v);
 445 }
 446
 447 static __always_inline int
 448 atomic_fetch_xor_acquire(int i, atomic_t *v)
 449 {
 450         instrument_atomic_read_write(v, sizeof(*v));
 451         return arch_atomic_fetch_xor_acquire(i, v);
 452 }
 453
 454 static __always_inline int
 455 atomic_fetch_xor_release(int i, atomic_t *v)
 456 {
 457         kcsan_release();
 458         instrument_atomic_read_write(v, sizeof(*v));
 459         return arch_atomic_fetch_xor_release(i, v);
 460 }
 461
 462 static __always_inline int
 463 atomic_fetch_xor_relaxed(int i, atomic_t *v)
 464 {
 465         instrument_atomic_read_write(v, sizeof(*v));
 466         return arch_atomic_fetch_xor_relaxed(i, v);
 467 }
 468
 469 static __always_inline int
 470 atomic_xchg(atomic_t *v, int i)
 471 {
 472         kcsan_mb();
 473         instrument_atomic_read_write(v, sizeof(*v));
 474         return arch_atomic_xchg(v, i);
 475 }
 476
 477 static __always_inline int
 478 atomic_xchg_acquire(atomic_t *v, int i)
 479 {
 480         instrument_atomic_read_write(v, sizeof(*v));
 481         return arch_atomic_xchg_acquire(v, i);
 482 }
 483
 484 static __always_inline int
 485 atomic_xchg_release(atomic_t *v, int i)
 486 {
 487         kcsan_release();
 488         instrument_atomic_read_write(v, sizeof(*v));
 489         return arch_atomic_xchg_release(v, i);
 490 }
 491
 492 static __always_inline int
 493 atomic_xchg_relaxed(atomic_t *v, int i)
 494 {
 495         instrument_atomic_read_write(v, sizeof(*v));
 496         return arch_atomic_xchg_relaxed(v, i);
 497 }
 498
 499 static __always_inline int
 500 atomic_cmpxchg(atomic_t *v, int old, int new)
 501 {
 502         kcsan_mb();
 503         instrument_atomic_read_write(v, sizeof(*v));
 504         return arch_atomic_cmpxchg(v, old, new);
 505 }
 506
 507 static __always_inline int
 508 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
 509 {
 510         instrument_atomic_read_write(v, sizeof(*v));
 511         return arch_atomic_cmpxchg_acquire(v, old, new);
 512 }
 513
 514 static __always_inline int
 515 atomic_cmpxchg_release(atomic_t *v, int old, int new)
 516 {
 517         kcsan_release();
 518         instrument_atomic_read_write(v, sizeof(*v));
 519         return arch_atomic_cmpxchg_release(v, old, new);
 520 }
 521
 522 static __always_inline int
 523 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
 524 {
 525         instrument_atomic_read_write(v, sizeof(*v));
 526         return arch_atomic_cmpxchg_relaxed(v, old, new);
 527 }
 528
 529 static __always_inline bool
 530 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
 531 {
 532         kcsan_mb();
 533         instrument_atomic_read_write(v, sizeof(*v));
 534         instrument_atomic_read_write(old, sizeof(*old));
 535         return arch_atomic_try_cmpxchg(v, old, new);
 536 }
 537
 538 static __always_inline bool
 539 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
 540 {
 541         instrument_atomic_read_write(v, sizeof(*v));
 542         instrument_atomic_read_write(old, sizeof(*old));
 543         return arch_atomic_try_cmpxchg_acquire(v, old, new);
 544 }
 545
 546 static __always_inline bool
 547 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
 548 {
 549         kcsan_release();
 550         instrument_atomic_read_write(v, sizeof(*v));
 551         instrument_atomic_read_write(old, sizeof(*old));
 552         return arch_atomic_try_cmpxchg_release(v, old, new);
 553 }
 554
 555 static __always_inline bool
 556 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
 557 {
 558         instrument_atomic_read_write(v, sizeof(*v));
 559         instrument_atomic_read_write(old, sizeof(*old));
 560         return arch_atomic_try_cmpxchg_relaxed(v, old, new);
 561 }
 562
 563 static __always_inline bool
 564 atomic_sub_and_test(int i, atomic_t *v)
 565 {
 566         kcsan_mb();
 567         instrument_atomic_read_write(v, sizeof(*v));
 568         return arch_atomic_sub_and_test(i, v);
 569 }
 570
 571 static __always_inline bool
 572 atomic_dec_and_test(atomic_t *v)
 573 {
 574         kcsan_mb();
 575         instrument_atomic_read_write(v, sizeof(*v));
 576         return arch_atomic_dec_and_test(v);
 577 }
 578
 579 static __always_inline bool
 580 atomic_inc_and_test(atomic_t *v)
 581 {
 582         kcsan_mb();
 583         instrument_atomic_read_write(v, sizeof(*v));
 584         return arch_atomic_inc_and_test(v);
 585 }
 586
 587 static __always_inline bool
 588 atomic_add_negative(int i, atomic_t *v)
 589 {
 590         kcsan_mb();
 591         instrument_atomic_read_write(v, sizeof(*v));
 592         return arch_atomic_add_negative(i, v);
 593 }
 594
 595 static __always_inline int
 596 atomic_fetch_add_unless(atomic_t *v, int a, int u)
 597 {
 598         kcsan_mb();
 599         instrument_atomic_read_write(v, sizeof(*v));
 600         return arch_atomic_fetch_add_unless(v, a, u);
 601 }
 602
 603 static __always_inline bool
 604 atomic_add_unless(atomic_t *v, int a, int u)
 605 {
 606         kcsan_mb();
 607         instrument_atomic_read_write(v, sizeof(*v));
 608         return arch_atomic_add_unless(v, a, u);
 609 }
 610
 611 static __always_inline bool
 612 atomic_inc_not_zero(atomic_t *v)
 613 {
 614         kcsan_mb();
 615         instrument_atomic_read_write(v, sizeof(*v));
 616         return arch_atomic_inc_not_zero(v);
 617 }
 618
 619 static __always_inline bool
 620 atomic_inc_unless_negative(atomic_t *v)
 621 {
 622         kcsan_mb();
 623         instrument_atomic_read_write(v, sizeof(*v));
 624         return arch_atomic_inc_unless_negative(v);
 625 }
 626
 627 static __always_inline bool
 628 atomic_dec_unless_positive(atomic_t *v)
 629 {
 630         kcsan_mb();
 631         instrument_atomic_read_write(v, sizeof(*v));
 632         return arch_atomic_dec_unless_positive(v);
 633 }
 634
 635 static __always_inline int
 636 atomic_dec_if_positive(atomic_t *v)
 637 {
 638         kcsan_mb();
 639         instrument_atomic_read_write(v, sizeof(*v));
 640         return arch_atomic_dec_if_positive(v);
 641 }
 642
 643 static __always_inline s64
 644 atomic64_read(const atomic64_t *v)
 645 {
 646         instrument_atomic_read(v, sizeof(*v));
 647         return arch_atomic64_read(v);
 648 }
 649
 650 static __always_inline s64
 651 atomic64_read_acquire(const atomic64_t *v)
 652 {
 653         instrument_atomic_read(v, sizeof(*v));
 654         return arch_atomic64_read_acquire(v);
 655 }
 656
 657 static __always_inline void
 658 atomic64_set(atomic64_t *v, s64 i)
 659 {
 660         instrument_atomic_write(v, sizeof(*v));
 661         arch_atomic64_set(v, i);
 662 }
 663
 664 static __always_inline void
 665 atomic64_set_release(atomic64_t *v, s64 i)
 666 {
 667         kcsan_release();
 668         instrument_atomic_write(v, sizeof(*v));
 669         arch_atomic64_set_release(v, i);
 670 }
 671
 672 static __always_inline void
 673 atomic64_add(s64 i, atomic64_t *v)
 674 {
 675         instrument_atomic_read_write(v, sizeof(*v));
 676         arch_atomic64_add(i, v);
 677 }
 678
 679 static __always_inline s64
 680 atomic64_add_return(s64 i, atomic64_t *v)
 681 {
 682         kcsan_mb();
 683         instrument_atomic_read_write(v, sizeof(*v));
 684         return arch_atomic64_add_return(i, v);
 685 }
 686
 687 static __always_inline s64
 688 atomic64_add_return_acquire(s64 i, atomic64_t *v)
 689 {
 690         instrument_atomic_read_write(v, sizeof(*v));
 691         return arch_atomic64_add_return_acquire(i, v);
 692 }
 693
 694 static __always_inline s64
 695 atomic64_add_return_release(s64 i, atomic64_t *v)
 696 {
 697         kcsan_release();
 698         instrument_atomic_read_write(v, sizeof(*v));
 699         return arch_atomic64_add_return_release(i, v);
 700 }
 701
 702 static __always_inline s64
 703 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
 704 {
 705         instrument_atomic_read_write(v, sizeof(*v));
 706         return arch_atomic64_add_return_relaxed(i, v);
 707 }
 708
 709 static __always_inline s64
 710 atomic64_fetch_add(s64 i, atomic64_t *v)
 711 {
 712         kcsan_mb();
 713         instrument_atomic_read_write(v, sizeof(*v));
 714         return arch_atomic64_fetch_add(i, v);
 715 }
 716
 717 static __always_inline s64
 718 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
 719 {
 720         instrument_atomic_read_write(v, sizeof(*v));
 721         return arch_atomic64_fetch_add_acquire(i, v);
 722 }
 723
 724 static __always_inline s64
 725 atomic64_fetch_add_release(s64 i, atomic64_t *v)
 726 {
 727         kcsan_release();
 728         instrument_atomic_read_write(v, sizeof(*v));
 729         return arch_atomic64_fetch_add_release(i, v);
 730 }
 731
 732 static __always_inline s64
 733 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
 734 {
 735         instrument_atomic_read_write(v, sizeof(*v));
 736         return arch_atomic64_fetch_add_relaxed(i, v);
 737 }
 738
 739 static __always_inline void
 740 atomic64_sub(s64 i, atomic64_t *v)
 741 {
 742         instrument_atomic_read_write(v, sizeof(*v));
 743         arch_atomic64_sub(i, v);
 744 }
 745
 746 static __always_inline s64
 747 atomic64_sub_return(s64 i, atomic64_t *v)
 748 {
 749         kcsan_mb();
 750         instrument_atomic_read_write(v, sizeof(*v));
 751         return arch_atomic64_sub_return(i, v);
 752 }
 753
 754 static __always_inline s64
 755 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
 756 {
 757         instrument_atomic_read_write(v, sizeof(*v));
 758         return arch_atomic64_sub_return_acquire(i, v);
 759 }
 760
 761 static __always_inline s64
 762 atomic64_sub_return_release(s64 i, atomic64_t *v)
 763 {
 764         kcsan_release();
 765         instrument_atomic_read_write(v, sizeof(*v));
 766         return arch_atomic64_sub_return_release(i, v);
 767 }
 768
 769 static __always_inline s64
 770 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
 771 {
 772         instrument_atomic_read_write(v, sizeof(*v));
 773         return arch_atomic64_sub_return_relaxed(i, v);
 774 }
 775
 776 static __always_inline s64
 777 atomic64_fetch_sub(s64 i, atomic64_t *v)
 778 {
 779         kcsan_mb();
 780         instrument_atomic_read_write(v, sizeof(*v));
 781         return arch_atomic64_fetch_sub(i, v);
 782 }
 783
 784 static __always_inline s64
 785 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
 786 {
 787         instrument_atomic_read_write(v, sizeof(*v));
 788         return arch_atomic64_fetch_sub_acquire(i, v);
 789 }
 790
 791 static __always_inline s64
 792 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
 793 {
 794         kcsan_release();
 795         instrument_atomic_read_write(v, sizeof(*v));
 796         return arch_atomic64_fetch_sub_release(i, v);
 797 }
 798
 799 static __always_inline s64
 800 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
 801 {
 802         instrument_atomic_read_write(v, sizeof(*v));
 803         return arch_atomic64_fetch_sub_relaxed(i, v);
 804 }
 805
 806 static __always_inline void
 807 atomic64_inc(atomic64_t *v)
 808 {
 809         instrument_atomic_read_write(v, sizeof(*v));
 810         arch_atomic64_inc(v);
 811 }
 812
 813 static __always_inline s64
 814 atomic64_inc_return(atomic64_t *v)
 815 {
 816         kcsan_mb();
 817         instrument_atomic_read_write(v, sizeof(*v));
 818         return arch_atomic64_inc_return(v);
 819 }
 820
 821 static __always_inline s64
 822 atomic64_inc_return_acquire(atomic64_t *v)
 823 {
 824         instrument_atomic_read_write(v, sizeof(*v));
 825         return arch_atomic64_inc_return_acquire(v);
 826 }
 827
 828 static __always_inline s64
 829 atomic64_inc_return_release(atomic64_t *v)
 830 {
 831         kcsan_release();
 832         instrument_atomic_read_write(v, sizeof(*v));
 833         return arch_atomic64_inc_return_release(v);
 834 }
 835
 836 static __always_inline s64
 837 atomic64_inc_return_relaxed(atomic64_t *v)
 838 {
 839         instrument_atomic_read_write(v, sizeof(*v));
 840         return arch_atomic64_inc_return_relaxed(v);
 841 }
 842
 843 static __always_inline s64
 844 atomic64_fetch_inc(atomic64_t *v)
 845 {
 846         kcsan_mb();
 847         instrument_atomic_read_write(v, sizeof(*v));
 848         return arch_atomic64_fetch_inc(v);
 849 }
 850
 851 static __always_inline s64
 852 atomic64_fetch_inc_acquire(atomic64_t *v)
 853 {
 854         instrument_atomic_read_write(v, sizeof(*v));
 855         return arch_atomic64_fetch_inc_acquire(v);
 856 }
 857
 858 static __always_inline s64
 859 atomic64_fetch_inc_release(atomic64_t *v)
 860 {
 861         kcsan_release();
 862         instrument_atomic_read_write(v, sizeof(*v));
 863         return arch_atomic64_fetch_inc_release(v);
 864 }
 865
 866 static __always_inline s64
 867 atomic64_fetch_inc_relaxed(atomic64_t *v)
 868 {
 869         instrument_atomic_read_write(v, sizeof(*v));
 870         return arch_atomic64_fetch_inc_relaxed(v);
 871 }
 872
 873 static __always_inline void
 874 atomic64_dec(atomic64_t *v)
 875 {
 876         instrument_atomic_read_write(v, sizeof(*v));
 877         arch_atomic64_dec(v);
 878 }
 879
 880 static __always_inline s64
 881 atomic64_dec_return(atomic64_t *v)
 882 {
 883         kcsan_mb();
 884         instrument_atomic_read_write(v, sizeof(*v));
 885         return arch_atomic64_dec_return(v);
 886 }
 887
 888 static __always_inline s64
 889 atomic64_dec_return_acquire(atomic64_t *v)
 890 {
 891         instrument_atomic_read_write(v, sizeof(*v));
 892         return arch_atomic64_dec_return_acquire(v);
 893 }
 894
 895 static __always_inline s64
 896 atomic64_dec_return_release(atomic64_t *v)
 897 {
 898         kcsan_release();
 899         instrument_atomic_read_write(v, sizeof(*v));
 900         return arch_atomic64_dec_return_release(v);
 901 }
 902
 903 static __always_inline s64
 904 atomic64_dec_return_relaxed(atomic64_t *v)
 905 {
 906         instrument_atomic_read_write(v, sizeof(*v));
 907         return arch_atomic64_dec_return_relaxed(v);
 908 }
 909
 910 static __always_inline s64
 911 atomic64_fetch_dec(atomic64_t *v)
 912 {
 913         kcsan_mb();
 914         instrument_atomic_read_write(v, sizeof(*v));
 915         return arch_atomic64_fetch_dec(v);
 916 }
 917
 918 static __always_inline s64
 919 atomic64_fetch_dec_acquire(atomic64_t *v)
 920 {
 921         instrument_atomic_read_write(v, sizeof(*v));
 922         return arch_atomic64_fetch_dec_acquire(v);
 923 }
 924
 925 static __always_inline s64
 926 atomic64_fetch_dec_release(atomic64_t *v)
 927 {
 928         kcsan_release();
 929         instrument_atomic_read_write(v, sizeof(*v));
 930         return arch_atomic64_fetch_dec_release(v);
 931 }
 932
 933 static __always_inline s64
 934 atomic64_fetch_dec_relaxed(atomic64_t *v)
 935 {
 936         instrument_atomic_read_write(v, sizeof(*v));
 937         return arch_atomic64_fetch_dec_relaxed(v);
 938 }
 939
 940 static __always_inline void
 941 atomic64_and(s64 i, atomic64_t *v)
 942 {
 943         instrument_atomic_read_write(v, sizeof(*v));
 944         arch_atomic64_and(i, v);
 945 }
 946
 947 static __always_inline s64
 948 atomic64_fetch_and(s64 i, atomic64_t *v)
 949 {
 950         kcsan_mb();
 951         instrument_atomic_read_write(v, sizeof(*v));
 952         return arch_atomic64_fetch_and(i, v);
 953 }
 954
 955 static __always_inline s64
 956 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
 957 {
 958         instrument_atomic_read_write(v, sizeof(*v));
 959         return arch_atomic64_fetch_and_acquire(i, v);
 960 }
 961
 962 static __always_inline s64
 963 atomic64_fetch_and_release(s64 i, atomic64_t *v)
 964 {
 965         kcsan_release();
 966         instrument_atomic_read_write(v, sizeof(*v));
 967         return arch_atomic64_fetch_and_release(i, v);
 968 }
 969
 970 static __always_inline s64
 971 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
 972 {
 973         instrument_atomic_read_write(v, sizeof(*v));
 974         return arch_atomic64_fetch_and_relaxed(i, v);
 975 }
 976
 977 static __always_inline void
 978 atomic64_andnot(s64 i, atomic64_t *v)
 979 {
 980         instrument_atomic_read_write(v, sizeof(*v));
 981         arch_atomic64_andnot(i, v);
 982 }
 983
 984 static __always_inline s64
 985 atomic64_fetch_andnot(s64 i, atomic64_t *v)
 986 {
 987         kcsan_mb();
 988         instrument_atomic_read_write(v, sizeof(*v));
 989         return arch_atomic64_fetch_andnot(i, v);
 990 }
 991
 992 static __always_inline s64
 993 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
 994 {
 995         instrument_atomic_read_write(v, sizeof(*v));
 996         return arch_atomic64_fetch_andnot_acquire(i, v);
 997 }
 998
 999 static __always_inline s64
1000 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1001 {
1002         kcsan_release();
1003         instrument_atomic_read_write(v, sizeof(*v));
1004         return arch_atomic64_fetch_andnot_release(i, v);
1005 }
1006
1007 static __always_inline s64
1008 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1009 {
1010         instrument_atomic_read_write(v, sizeof(*v));
1011         return arch_atomic64_fetch_andnot_relaxed(i, v);
1012 }
1013
1014 static __always_inline void
1015 atomic64_or(s64 i, atomic64_t *v)
1016 {
1017         instrument_atomic_read_write(v, sizeof(*v));
1018         arch_atomic64_or(i, v);
1019 }
1020
1021 static __always_inline s64
1022 atomic64_fetch_or(s64 i, atomic64_t *v)
1023 {
1024         kcsan_mb();
1025         instrument_atomic_read_write(v, sizeof(*v));
1026         return arch_atomic64_fetch_or(i, v);
1027 }
1028
1029 static __always_inline s64
1030 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1031 {
1032         instrument_atomic_read_write(v, sizeof(*v));
1033         return arch_atomic64_fetch_or_acquire(i, v);
1034 }
1035
1036 static __always_inline s64
1037 atomic64_fetch_or_release(s64 i, atomic64_t *v)
1038 {
1039         kcsan_release();
1040         instrument_atomic_read_write(v, sizeof(*v));
1041         return arch_atomic64_fetch_or_release(i, v);
1042 }
1043
1044 static __always_inline s64
1045 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1046 {
1047         instrument_atomic_read_write(v, sizeof(*v));
1048         return arch_atomic64_fetch_or_relaxed(i, v);
1049 }
1050
1051 static __always_inline void
1052 atomic64_xor(s64 i, atomic64_t *v)
1053 {
1054         instrument_atomic_read_write(v, sizeof(*v));
1055         arch_atomic64_xor(i, v);
1056 }
1057
1058 static __always_inline s64
1059 atomic64_fetch_xor(s64 i, atomic64_t *v)
1060 {
1061         kcsan_mb();
1062         instrument_atomic_read_write(v, sizeof(*v));
1063         return arch_atomic64_fetch_xor(i, v);
1064 }
1065
1066 static __always_inline s64
1067 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1068 {
1069         instrument_atomic_read_write(v, sizeof(*v));
1070         return arch_atomic64_fetch_xor_acquire(i, v);
1071 }
1072
1073 static __always_inline s64
1074 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1075 {
1076         kcsan_release();
1077         instrument_atomic_read_write(v, sizeof(*v));
1078         return arch_atomic64_fetch_xor_release(i, v);
1079 }
1080
1081 static __always_inline s64
1082 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1083 {
1084         instrument_atomic_read_write(v, sizeof(*v));
1085         return arch_atomic64_fetch_xor_relaxed(i, v);
1086 }
1087
1088 static __always_inline s64
1089 atomic64_xchg(atomic64_t *v, s64 i)
1090 {
1091         kcsan_mb();
1092         instrument_atomic_read_write(v, sizeof(*v));
1093         return arch_atomic64_xchg(v, i);
1094 }
1095
1096 static __always_inline s64
1097 atomic64_xchg_acquire(atomic64_t *v, s64 i)
1098 {
1099         instrument_atomic_read_write(v, sizeof(*v));
1100         return arch_atomic64_xchg_acquire(v, i);
1101 }
1102
1103 static __always_inline s64
1104 atomic64_xchg_release(atomic64_t *v, s64 i)
1105 {
1106         kcsan_release();
1107         instrument_atomic_read_write(v, sizeof(*v));
1108         return arch_atomic64_xchg_release(v, i);
1109 }
1110
1111 static __always_inline s64
1112 atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1113 {
1114         instrument_atomic_read_write(v, sizeof(*v));
1115         return arch_atomic64_xchg_relaxed(v, i);
1116 }
1117
1118 static __always_inline s64
1119 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1120 {
1121         kcsan_mb();
1122         instrument_atomic_read_write(v, sizeof(*v));
1123         return arch_atomic64_cmpxchg(v, old, new);
1124 }
1125
1126 static __always_inline s64
1127 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1128 {
1129         instrument_atomic_read_write(v, sizeof(*v));
1130         return arch_atomic64_cmpxchg_acquire(v, old, new);
1131 }
1132
1133 static __always_inline s64
1134 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1135 {
1136         kcsan_release();
1137         instrument_atomic_read_write(v, sizeof(*v));
1138         return arch_atomic64_cmpxchg_release(v, old, new);
1139 }
1140
1141 static __always_inline s64
1142 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1143 {
1144         instrument_atomic_read_write(v, sizeof(*v));
1145         return arch_atomic64_cmpxchg_relaxed(v, old, new);
1146 }
1147
1148 static __always_inline bool
1149 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1150 {
1151         kcsan_mb();
1152         instrument_atomic_read_write(v, sizeof(*v));
1153         instrument_atomic_read_write(old, sizeof(*old));
1154         return arch_atomic64_try_cmpxchg(v, old, new);
1155 }
1156
1157 static __always_inline bool
1158 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1159 {
1160         instrument_atomic_read_write(v, sizeof(*v));
1161         instrument_atomic_read_write(old, sizeof(*old));
1162         return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1163 }
1164
1165 static __always_inline bool
1166 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1167 {
1168         kcsan_release();
1169         instrument_atomic_read_write(v, sizeof(*v));
1170         instrument_atomic_read_write(old, sizeof(*old));
1171         return arch_atomic64_try_cmpxchg_release(v, old, new);
1172 }
1173
1174 static __always_inline bool
1175 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1176 {
1177         instrument_atomic_read_write(v, sizeof(*v));
1178         instrument_atomic_read_write(old, sizeof(*old));
1179         return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1180 }
1181
1182 static __always_inline bool
1183 atomic64_sub_and_test(s64 i, atomic64_t *v)
1184 {
1185         kcsan_mb();
1186         instrument_atomic_read_write(v, sizeof(*v));
1187         return arch_atomic64_sub_and_test(i, v);
1188 }
1189
1190 static __always_inline bool
1191 atomic64_dec_and_test(atomic64_t *v)
1192 {
1193         kcsan_mb();
1194         instrument_atomic_read_write(v, sizeof(*v));
1195         return arch_atomic64_dec_and_test(v);
1196 }
1197
1198 static __always_inline bool
1199 atomic64_inc_and_test(atomic64_t *v)
1200 {
1201         kcsan_mb();
1202         instrument_atomic_read_write(v, sizeof(*v));
1203         return arch_atomic64_inc_and_test(v);
1204 }
1205
1206 static __always_inline bool
1207 atomic64_add_negative(s64 i, atomic64_t *v)
1208 {
1209         kcsan_mb();
1210         instrument_atomic_read_write(v, sizeof(*v));
1211         return arch_atomic64_add_negative(i, v);
1212 }
1213
1214 static __always_inline s64
1215 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1216 {
1217         kcsan_mb();
1218         instrument_atomic_read_write(v, sizeof(*v));
1219         return arch_atomic64_fetch_add_unless(v, a, u);
1220 }
1221
1222 static __always_inline bool
1223 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1224 {
1225         kcsan_mb();
1226         instrument_atomic_read_write(v, sizeof(*v));
1227         return arch_atomic64_add_unless(v, a, u);
1228 }
1229
1230 static __always_inline bool
1231 atomic64_inc_not_zero(atomic64_t *v)
1232 {
1233         kcsan_mb();
1234         instrument_atomic_read_write(v, sizeof(*v));
1235         return arch_atomic64_inc_not_zero(v);
1236 }
1237
1238 static __always_inline bool
1239 atomic64_inc_unless_negative(atomic64_t *v)
1240 {
1241         kcsan_mb();
1242         instrument_atomic_read_write(v, sizeof(*v));
1243         return arch_atomic64_inc_unless_negative(v);
1244 }
1245
1246 static __always_inline bool
1247 atomic64_dec_unless_positive(atomic64_t *v)
1248 {
1249         kcsan_mb();
1250         instrument_atomic_read_write(v, sizeof(*v));
1251         return arch_atomic64_dec_unless_positive(v);
1252 }
1253
1254 static __always_inline s64
1255 atomic64_dec_if_positive(atomic64_t *v)
1256 {
1257         kcsan_mb();
1258         instrument_atomic_read_write(v, sizeof(*v));
1259         return arch_atomic64_dec_if_positive(v);
1260 }
1261
1262 static __always_inline long
1263 atomic_long_read(const atomic_long_t *v)
1264 {
1265         instrument_atomic_read(v, sizeof(*v));
1266         return arch_atomic_long_read(v);
1267 }
1268
1269 static __always_inline long
1270 atomic_long_read_acquire(const atomic_long_t *v)
1271 {
1272         instrument_atomic_read(v, sizeof(*v));
1273         return arch_atomic_long_read_acquire(v);
1274 }
1275
1276 static __always_inline void
1277 atomic_long_set(atomic_long_t *v, long i)
1278 {
1279         instrument_atomic_write(v, sizeof(*v));
1280         arch_atomic_long_set(v, i);
1281 }
1282
1283 static __always_inline void
1284 atomic_long_set_release(atomic_long_t *v, long i)
1285 {
1286         kcsan_release();
1287         instrument_atomic_write(v, sizeof(*v));
1288         arch_atomic_long_set_release(v, i);
1289 }
1290
1291 static __always_inline void
1292 atomic_long_add(long i, atomic_long_t *v)
1293 {
1294         instrument_atomic_read_write(v, sizeof(*v));
1295         arch_atomic_long_add(i, v);
1296 }
1297
1298 static __always_inline long
1299 atomic_long_add_return(long i, atomic_long_t *v)
1300 {
1301         kcsan_mb();
1302         instrument_atomic_read_write(v, sizeof(*v));
1303         return arch_atomic_long_add_return(i, v);
1304 }
1305
1306 static __always_inline long
1307 atomic_long_add_return_acquire(long i, atomic_long_t *v)
1308 {
1309         instrument_atomic_read_write(v, sizeof(*v));
1310         return arch_atomic_long_add_return_acquire(i, v);
1311 }
1312
1313 static __always_inline long
1314 atomic_long_add_return_release(long i, atomic_long_t *v)
1315 {
1316         kcsan_release();
1317         instrument_atomic_read_write(v, sizeof(*v));
1318         return arch_atomic_long_add_return_release(i, v);
1319 }
1320
1321 static __always_inline long
1322 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1323 {
1324         instrument_atomic_read_write(v, sizeof(*v));
1325         return arch_atomic_long_add_return_relaxed(i, v);
1326 }
1327
1328 static __always_inline long
1329 atomic_long_fetch_add(long i, atomic_long_t *v)
1330 {
1331         kcsan_mb();
1332         instrument_atomic_read_write(v, sizeof(*v));
1333         return arch_atomic_long_fetch_add(i, v);
1334 }
1335
1336 static __always_inline long
1337 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1338 {
1339         instrument_atomic_read_write(v, sizeof(*v));
1340         return arch_atomic_long_fetch_add_acquire(i, v);
1341 }
1342
1343 static __always_inline long
1344 atomic_long_fetch_add_release(long i, atomic_long_t *v)
1345 {
1346         kcsan_release();
1347         instrument_atomic_read_write(v, sizeof(*v));
1348         return arch_atomic_long_fetch_add_release(i, v);
1349 }
1350
1351 static __always_inline long
1352 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1353 {
1354         instrument_atomic_read_write(v, sizeof(*v));
1355         return arch_atomic_long_fetch_add_relaxed(i, v);
1356 }
1357
1358 static __always_inline void
1359 atomic_long_sub(long i, atomic_long_t *v)
1360 {
1361         instrument_atomic_read_write(v, sizeof(*v));
1362         arch_atomic_long_sub(i, v);
1363 }
1364
1365 static __always_inline long
1366 atomic_long_sub_return(long i, atomic_long_t *v)
1367 {
1368         kcsan_mb();
1369         instrument_atomic_read_write(v, sizeof(*v));
1370         return arch_atomic_long_sub_return(i, v);
1371 }
1372
1373 static __always_inline long
1374 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1375 {
1376         instrument_atomic_read_write(v, sizeof(*v));
1377         return arch_atomic_long_sub_return_acquire(i, v);
1378 }
1379
1380 static __always_inline long
1381 atomic_long_sub_return_release(long i, atomic_long_t *v)
1382 {
1383         kcsan_release();
1384         instrument_atomic_read_write(v, sizeof(*v));
1385         return arch_atomic_long_sub_return_release(i, v);
1386 }
1387
1388 static __always_inline long
1389 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1390 {
1391         instrument_atomic_read_write(v, sizeof(*v));
1392         return arch_atomic_long_sub_return_relaxed(i, v);
1393 }
1394
1395 static __always_inline long
1396 atomic_long_fetch_sub(long i, atomic_long_t *v)
1397 {
1398         kcsan_mb();
1399         instrument_atomic_read_write(v, sizeof(*v));
1400         return arch_atomic_long_fetch_sub(i, v);
1401 }
1402
1403 static __always_inline long
1404 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1405 {
1406         instrument_atomic_read_write(v, sizeof(*v));
1407         return arch_atomic_long_fetch_sub_acquire(i, v);
1408 }
1409
1410 static __always_inline long
1411 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1412 {
1413         kcsan_release();
1414         instrument_atomic_read_write(v, sizeof(*v));
1415         return arch_atomic_long_fetch_sub_release(i, v);
1416 }
1417
1418 static __always_inline long
1419 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1420 {
1421         instrument_atomic_read_write(v, sizeof(*v));
1422         return arch_atomic_long_fetch_sub_relaxed(i, v);
1423 }
1424
1425 static __always_inline void
1426 atomic_long_inc(atomic_long_t *v)
1427 {
1428         instrument_atomic_read_write(v, sizeof(*v));
1429         arch_atomic_long_inc(v);
1430 }
1431
1432 static __always_inline long
1433 atomic_long_inc_return(atomic_long_t *v)
1434 {
1435         kcsan_mb();
1436         instrument_atomic_read_write(v, sizeof(*v));
1437         return arch_atomic_long_inc_return(v);
1438 }
1439
1440 static __always_inline long
1441 atomic_long_inc_return_acquire(atomic_long_t *v)
1442 {
1443         instrument_atomic_read_write(v, sizeof(*v));
1444         return arch_atomic_long_inc_return_acquire(v);
1445 }
1446
1447 static __always_inline long
1448 atomic_long_inc_return_release(atomic_long_t *v)
1449 {
1450         kcsan_release();
1451         instrument_atomic_read_write(v, sizeof(*v));
1452         return arch_atomic_long_inc_return_release(v);
1453 }
1454
1455 static __always_inline long
1456 atomic_long_inc_return_relaxed(atomic_long_t *v)
1457 {
1458         instrument_atomic_read_write(v, sizeof(*v));
1459         return arch_atomic_long_inc_return_relaxed(v);
1460 }
1461
1462 static __always_inline long
1463 atomic_long_fetch_inc(atomic_long_t *v)
1464 {
1465         kcsan_mb();
1466         instrument_atomic_read_write(v, sizeof(*v));
1467         return arch_atomic_long_fetch_inc(v);
1468 }
1469
1470 static __always_inline long
1471 atomic_long_fetch_inc_acquire(atomic_long_t *v)
1472 {
1473         instrument_atomic_read_write(v, sizeof(*v));
1474         return arch_atomic_long_fetch_inc_acquire(v);
1475 }
1476
1477 static __always_inline long
1478 atomic_long_fetch_inc_release(atomic_long_t *v)
1479 {
1480         kcsan_release();
1481         instrument_atomic_read_write(v, sizeof(*v));
1482         return arch_atomic_long_fetch_inc_release(v);
1483 }
1484
1485 static __always_inline long
1486 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1487 {
1488         instrument_atomic_read_write(v, sizeof(*v));
1489         return arch_atomic_long_fetch_inc_relaxed(v);
1490 }
1491
1492 static __always_inline void
1493 atomic_long_dec(atomic_long_t *v)
1494 {
1495         instrument_atomic_read_write(v, sizeof(*v));
1496         arch_atomic_long_dec(v);
1497 }
1498
1499 static __always_inline long
1500 atomic_long_dec_return(atomic_long_t *v)
1501 {
1502         kcsan_mb();
1503         instrument_atomic_read_write(v, sizeof(*v));
1504         return arch_atomic_long_dec_return(v);
1505 }
1506
1507 static __always_inline long
1508 atomic_long_dec_return_acquire(atomic_long_t *v)
1509 {
1510         instrument_atomic_read_write(v, sizeof(*v));
1511         return arch_atomic_long_dec_return_acquire(v);
1512 }
1513
1514 static __always_inline long
1515 atomic_long_dec_return_release(atomic_long_t *v)
1516 {
1517         kcsan_release();
1518         instrument_atomic_read_write(v, sizeof(*v));
1519         return arch_atomic_long_dec_return_release(v);
1520 }
1521
1522 static __always_inline long
1523 atomic_long_dec_return_relaxed(atomic_long_t *v)
1524 {
1525         instrument_atomic_read_write(v, sizeof(*v));
1526         return arch_atomic_long_dec_return_relaxed(v);
1527 }
1528
1529 static __always_inline long
1530 atomic_long_fetch_dec(atomic_long_t *v)
1531 {
1532         kcsan_mb();
1533         instrument_atomic_read_write(v, sizeof(*v));
1534         return arch_atomic_long_fetch_dec(v);
1535 }
1536
1537 static __always_inline long
1538 atomic_long_fetch_dec_acquire(atomic_long_t *v)
1539 {
1540         instrument_atomic_read_write(v, sizeof(*v));
1541         return arch_atomic_long_fetch_dec_acquire(v);
1542 }
1543
1544 static __always_inline long
1545 atomic_long_fetch_dec_release(atomic_long_t *v)
1546 {
1547         kcsan_release();
1548         instrument_atomic_read_write(v, sizeof(*v));
1549         return arch_atomic_long_fetch_dec_release(v);
1550 }
1551
1552 static __always_inline long
1553 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1554 {
1555         instrument_atomic_read_write(v, sizeof(*v));
1556         return arch_atomic_long_fetch_dec_relaxed(v);
1557 }
1558
1559 static __always_inline void
1560 atomic_long_and(long i, atomic_long_t *v)
1561 {
1562         instrument_atomic_read_write(v, sizeof(*v));
1563         arch_atomic_long_and(i, v);
1564 }
1565
1566 static __always_inline long
1567 atomic_long_fetch_and(long i, atomic_long_t *v)
1568 {
1569         kcsan_mb();
1570         instrument_atomic_read_write(v, sizeof(*v));
1571         return arch_atomic_long_fetch_and(i, v);
1572 }
1573
1574 static __always_inline long
1575 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1576 {
1577         instrument_atomic_read_write(v, sizeof(*v));
1578         return arch_atomic_long_fetch_and_acquire(i, v);
1579 }
1580
1581 static __always_inline long
1582 atomic_long_fetch_and_release(long i, atomic_long_t *v)
1583 {
1584         kcsan_release();
1585         instrument_atomic_read_write(v, sizeof(*v));
1586         return arch_atomic_long_fetch_and_release(i, v);
1587 }
1588
1589 static __always_inline long
1590 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1591 {
1592         instrument_atomic_read_write(v, sizeof(*v));
1593         return arch_atomic_long_fetch_and_relaxed(i, v);
1594 }
1595
1596 static __always_inline void
1597 atomic_long_andnot(long i, atomic_long_t *v)
1598 {
1599         instrument_atomic_read_write(v, sizeof(*v));
1600         arch_atomic_long_andnot(i, v);
1601 }
1602
1603 static __always_inline long
1604 atomic_long_fetch_andnot(long i, atomic_long_t *v)
1605 {
1606         kcsan_mb();
1607         instrument_atomic_read_write(v, sizeof(*v));
1608         return arch_atomic_long_fetch_andnot(i, v);
1609 }
1610
1611 static __always_inline long
1612 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1613 {
1614         instrument_atomic_read_write(v, sizeof(*v));
1615         return arch_atomic_long_fetch_andnot_acquire(i, v);
1616 }
1617
1618 static __always_inline long
1619 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1620 {
1621         kcsan_release();
1622         instrument_atomic_read_write(v, sizeof(*v));
1623         return arch_atomic_long_fetch_andnot_release(i, v);
1624 }
1625
1626 static __always_inline long
1627 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1628 {
1629         instrument_atomic_read_write(v, sizeof(*v));
1630         return arch_atomic_long_fetch_andnot_relaxed(i, v);
1631 }
1632
1633 static __always_inline void
1634 atomic_long_or(long i, atomic_long_t *v)
1635 {
1636         instrument_atomic_read_write(v, sizeof(*v));
1637         arch_atomic_long_or(i, v);
1638 }
1639
1640 static __always_inline long
1641 atomic_long_fetch_or(long i, atomic_long_t *v)
1642 {
1643         kcsan_mb();
1644         instrument_atomic_read_write(v, sizeof(*v));
1645         return arch_atomic_long_fetch_or(i, v);
1646 }
1647
1648 static __always_inline long
1649 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1650 {
1651         instrument_atomic_read_write(v, sizeof(*v));
1652         return arch_atomic_long_fetch_or_acquire(i, v);
1653 }
1654
1655 static __always_inline long
1656 atomic_long_fetch_or_release(long i, atomic_long_t *v)
1657 {
1658         kcsan_release();
1659         instrument_atomic_read_write(v, sizeof(*v));
1660         return arch_atomic_long_fetch_or_release(i, v);
1661 }
1662
1663 static __always_inline long
1664 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1665 {
1666         instrument_atomic_read_write(v, sizeof(*v));
1667         return arch_atomic_long_fetch_or_relaxed(i, v);
1668 }
1669
1670 static __always_inline void
1671 atomic_long_xor(long i, atomic_long_t *v)
1672 {
1673         instrument_atomic_read_write(v, sizeof(*v));
1674         arch_atomic_long_xor(i, v);
1675 }
1676
1677 static __always_inline long
1678 atomic_long_fetch_xor(long i, atomic_long_t *v)
1679 {
1680         kcsan_mb();
1681         instrument_atomic_read_write(v, sizeof(*v));
1682         return arch_atomic_long_fetch_xor(i, v);
1683 }
1684
1685 static __always_inline long
1686 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1687 {
1688         instrument_atomic_read_write(v, sizeof(*v));
1689         return arch_atomic_long_fetch_xor_acquire(i, v);
1690 }
1691
1692 static __always_inline long
1693 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1694 {
1695         kcsan_release();
1696         instrument_atomic_read_write(v, sizeof(*v));
1697         return arch_atomic_long_fetch_xor_release(i, v);
1698 }
1699
1700 static __always_inline long
1701 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1702 {
1703         instrument_atomic_read_write(v, sizeof(*v));
1704         return arch_atomic_long_fetch_xor_relaxed(i, v);
1705 }
1706
1707 static __always_inline long
1708 atomic_long_xchg(atomic_long_t *v, long i)
1709 {
1710         kcsan_mb();
1711         instrument_atomic_read_write(v, sizeof(*v));
1712         return arch_atomic_long_xchg(v, i);
1713 }
1714
1715 static __always_inline long
1716 atomic_long_xchg_acquire(atomic_long_t *v, long i)
1717 {
1718         instrument_atomic_read_write(v, sizeof(*v));
1719         return arch_atomic_long_xchg_acquire(v, i);
1720 }
1721
1722 static __always_inline long
1723 atomic_long_xchg_release(atomic_long_t *v, long i)
1724 {
1725         kcsan_release();
1726         instrument_atomic_read_write(v, sizeof(*v));
1727         return arch_atomic_long_xchg_release(v, i);
1728 }
1729
1730 static __always_inline long
1731 atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1732 {
1733         instrument_atomic_read_write(v, sizeof(*v));
1734         return arch_atomic_long_xchg_relaxed(v, i);
1735 }
1736
1737 static __always_inline long
1738 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1739 {
1740         kcsan_mb();
1741         instrument_atomic_read_write(v, sizeof(*v));
1742         return arch_atomic_long_cmpxchg(v, old, new);
1743 }
1744
1745 static __always_inline long
1746 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1747 {
1748         instrument_atomic_read_write(v, sizeof(*v));
1749         return arch_atomic_long_cmpxchg_acquire(v, old, new);
1750 }
1751
1752 static __always_inline long
1753 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1754 {
1755         kcsan_release();
1756         instrument_atomic_read_write(v, sizeof(*v));
1757         return arch_atomic_long_cmpxchg_release(v, old, new);
1758 }
1759
1760 static __always_inline long
1761 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1762 {
1763         instrument_atomic_read_write(v, sizeof(*v));
1764         return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1765 }
1766
1767 static __always_inline bool
1768 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1769 {
1770         kcsan_mb();
1771         instrument_atomic_read_write(v, sizeof(*v));
1772         instrument_atomic_read_write(old, sizeof(*old));
1773         return arch_atomic_long_try_cmpxchg(v, old, new);
1774 }
1775
1776 static __always_inline bool
1777 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1778 {
1779         instrument_atomic_read_write(v, sizeof(*v));
1780         instrument_atomic_read_write(old, sizeof(*old));
1781         return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1782 }
1783
1784 static __always_inline bool
1785 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1786 {
1787         kcsan_release();
1788         instrument_atomic_read_write(v, sizeof(*v));
1789         instrument_atomic_read_write(old, sizeof(*old));
1790         return arch_atomic_long_try_cmpxchg_release(v, old, new);
1791 }
1792
1793 static __always_inline bool
1794 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1795 {
1796         instrument_atomic_read_write(v, sizeof(*v));
1797         instrument_atomic_read_write(old, sizeof(*old));
1798         return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1799 }
1800
1801 static __always_inline bool
1802 atomic_long_sub_and_test(long i, atomic_long_t *v)
1803 {
1804         kcsan_mb();
1805         instrument_atomic_read_write(v, sizeof(*v));
1806         return arch_atomic_long_sub_and_test(i, v);
1807 }
1808
1809 static __always_inline bool
1810 atomic_long_dec_and_test(atomic_long_t *v)
1811 {
1812         kcsan_mb();
1813         instrument_atomic_read_write(v, sizeof(*v));
1814         return arch_atomic_long_dec_and_test(v);
1815 }
1816
1817 static __always_inline bool
1818 atomic_long_inc_and_test(atomic_long_t *v)
1819 {
1820         kcsan_mb();
1821         instrument_atomic_read_write(v, sizeof(*v));
1822         return arch_atomic_long_inc_and_test(v);
1823 }
1824
1825 static __always_inline bool
1826 atomic_long_add_negative(long i, atomic_long_t *v)
1827 {
1828         kcsan_mb();
1829         instrument_atomic_read_write(v, sizeof(*v));
1830         return arch_atomic_long_add_negative(i, v);
1831 }
1832
1833 static __always_inline long
1834 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1835 {
1836         kcsan_mb();
1837         instrument_atomic_read_write(v, sizeof(*v));
1838         return arch_atomic_long_fetch_add_unless(v, a, u);
1839 }
1840
1841 static __always_inline bool
1842 atomic_long_add_unless(atomic_long_t *v, long a, long u)
1843 {
1844         kcsan_mb();
1845         instrument_atomic_read_write(v, sizeof(*v));
1846         return arch_atomic_long_add_unless(v, a, u);
1847 }
1848
1849 static __always_inline bool
1850 atomic_long_inc_not_zero(atomic_long_t *v)
1851 {
1852         kcsan_mb();
1853         instrument_atomic_read_write(v, sizeof(*v));
1854         return arch_atomic_long_inc_not_zero(v);
1855 }
1856
1857 static __always_inline bool
1858 atomic_long_inc_unless_negative(atomic_long_t *v)
1859 {
1860         kcsan_mb();
1861         instrument_atomic_read_write(v, sizeof(*v));
1862         return arch_atomic_long_inc_unless_negative(v);
1863 }
1864
1865 static __always_inline bool
1866 atomic_long_dec_unless_positive(atomic_long_t *v)
1867 {
1868         kcsan_mb();
1869         instrument_atomic_read_write(v, sizeof(*v));
1870         return arch_atomic_long_dec_unless_positive(v);
1871 }
1872
1873 static __always_inline long
1874 atomic_long_dec_if_positive(atomic_long_t *v)
1875 {
1876         kcsan_mb();
1877         instrument_atomic_read_write(v, sizeof(*v));
1878         return arch_atomic_long_dec_if_positive(v);
1879 }
1880
1881 #define xchg(ptr, ...) \
1882 ({ \
1883         typeof(ptr) __ai_ptr = (ptr); \
1884         kcsan_mb(); \
1885         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1886         arch_xchg(__ai_ptr, __VA_ARGS__); \
1887 })
1888
1889 #define xchg_acquire(ptr, ...) \
1890 ({ \
1891         typeof(ptr) __ai_ptr = (ptr); \
1892         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1893         arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1894 })
1895
1896 #define xchg_release(ptr, ...) \
1897 ({ \
1898         typeof(ptr) __ai_ptr = (ptr); \
1899         kcsan_release(); \
1900         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1901         arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1902 })
1903
1904 #define xchg_relaxed(ptr, ...) \
1905 ({ \
1906         typeof(ptr) __ai_ptr = (ptr); \
1907         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1908         arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1909 })
1910
1911 #define cmpxchg(ptr, ...) \
1912 ({ \
1913         typeof(ptr) __ai_ptr = (ptr); \
1914         kcsan_mb(); \
1915         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1916         arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1917 })
1918
1919 #define cmpxchg_acquire(ptr, ...) \
1920 ({ \
1921         typeof(ptr) __ai_ptr = (ptr); \
1922         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1923         arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1924 })
1925
1926 #define cmpxchg_release(ptr, ...) \
1927 ({ \
1928         typeof(ptr) __ai_ptr = (ptr); \
1929         kcsan_release(); \
1930         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1931         arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1932 })
1933
1934 #define cmpxchg_relaxed(ptr, ...) \
1935 ({ \
1936         typeof(ptr) __ai_ptr = (ptr); \
1937         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1938         arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1939 })
1940
1941 #define cmpxchg64(ptr, ...) \
1942 ({ \
1943         typeof(ptr) __ai_ptr = (ptr); \
1944         kcsan_mb(); \
1945         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1946         arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1947 })
1948
1949 #define cmpxchg64_acquire(ptr, ...) \
1950 ({ \
1951         typeof(ptr) __ai_ptr = (ptr); \
1952         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1953         arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1954 })
1955
1956 #define cmpxchg64_release(ptr, ...) \
1957 ({ \
1958         typeof(ptr) __ai_ptr = (ptr); \
1959         kcsan_release(); \
1960         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1961         arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1962 })
1963
1964 #define cmpxchg64_relaxed(ptr, ...) \
1965 ({ \
1966         typeof(ptr) __ai_ptr = (ptr); \
1967         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1968         arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1969 })
1970
1971 #define try_cmpxchg(ptr, oldp, ...) \
1972 ({ \
1973         typeof(ptr) __ai_ptr = (ptr); \
1974         typeof(oldp) __ai_oldp = (oldp); \
1975         kcsan_mb(); \
1976         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1977         instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1978         arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1979 })
1980
1981 #define try_cmpxchg_acquire(ptr, oldp, ...) \
1982 ({ \
1983         typeof(ptr) __ai_ptr = (ptr); \
1984         typeof(oldp) __ai_oldp = (oldp); \
1985         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1986         instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1987         arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1988 })
1989
1990 #define try_cmpxchg_release(ptr, oldp, ...) \
1991 ({ \
1992         typeof(ptr) __ai_ptr = (ptr); \
1993         typeof(oldp) __ai_oldp = (oldp); \
1994         kcsan_release(); \
1995         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1996         instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1997         arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1998 })
1999
2000 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
2001 ({ \
2002         typeof(ptr) __ai_ptr = (ptr); \
2003         typeof(oldp) __ai_oldp = (oldp); \
2004         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2005         instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2006         arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2007 })
2008
2009 #define try_cmpxchg64(ptr, oldp, ...) \
2010 ({ \
2011         typeof(ptr) __ai_ptr = (ptr); \
2012         typeof(oldp) __ai_oldp = (oldp); \
2013         kcsan_mb(); \
2014         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2015         instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2016         arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2017 })
2018
2019 #define try_cmpxchg64_acquire(ptr, oldp, ...) \
2020 ({ \
2021         typeof(ptr) __ai_ptr = (ptr); \
2022         typeof(oldp) __ai_oldp = (oldp); \
2023         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2024         instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2025         arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2026 })
2027
2028 #define try_cmpxchg64_release(ptr, oldp, ...) \
2029 ({ \
2030         typeof(ptr) __ai_ptr = (ptr); \
2031         typeof(oldp) __ai_oldp = (oldp); \
2032         kcsan_release(); \
2033         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034         instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2035         arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2036 })
2037
2038 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
2039 ({ \
2040         typeof(ptr) __ai_ptr = (ptr); \
2041         typeof(oldp) __ai_oldp = (oldp); \
2042         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043         instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044         arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045 })
2046
2047 #define cmpxchg_local(ptr, ...) \
2048 ({ \
2049         typeof(ptr) __ai_ptr = (ptr); \
2050         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2051         arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2052 })
2053
2054 #define cmpxchg64_local(ptr, ...) \
2055 ({ \
2056         typeof(ptr) __ai_ptr = (ptr); \
2057         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2058         arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2059 })
2060
2061 #define sync_cmpxchg(ptr, ...) \
2062 ({ \
2063         typeof(ptr) __ai_ptr = (ptr); \
2064         kcsan_mb(); \
2065         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2066         arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2067 })
2068
2069 #define cmpxchg_double(ptr, ...) \
2070 ({ \
2071         typeof(ptr) __ai_ptr = (ptr); \
2072         kcsan_mb(); \
2073         instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2074         arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2075 })
2076
2077
2078 #define cmpxchg_double_local(ptr, ...) \
2079 ({ \
2080         typeof(ptr) __ai_ptr = (ptr); \
2081         instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2082         arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2083 })
2084
2085 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2086 // 764f741eb77a7ad565dc8d99ce2837d5542e8aee