include/linux/atomic/atomic-arch-fallback.h

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 // Generated by scripts/atomic/gen-atomic-fallback.sh
   4 // DO NOT MODIFY THIS FILE DIRECTLY
   5
   6 #ifndef _LINUX_ATOMIC_FALLBACK_H
   7 #define _LINUX_ATOMIC_FALLBACK_H
   8
   9 #include <linux/compiler.h>
  10
  11 #ifndef arch_xchg_relaxed
  12 #define arch_xchg_acquire arch_xchg
  13 #define arch_xchg_release arch_xchg
  14 #define arch_xchg_relaxed arch_xchg
  15 #else /* arch_xchg_relaxed */
  16
  17 #ifndef arch_xchg_acquire
  18 #define arch_xchg_acquire(...) \
  19         __atomic_op_acquire(arch_xchg, __VA_ARGS__)
  20 #endif
  21
  22 #ifndef arch_xchg_release
  23 #define arch_xchg_release(...) \
  24         __atomic_op_release(arch_xchg, __VA_ARGS__)
  25 #endif
  26
  27 #ifndef arch_xchg
  28 #define arch_xchg(...) \
  29         __atomic_op_fence(arch_xchg, __VA_ARGS__)
  30 #endif
  31
  32 #endif /* arch_xchg_relaxed */
  33
  34 #ifndef arch_cmpxchg_relaxed
  35 #define arch_cmpxchg_acquire arch_cmpxchg
  36 #define arch_cmpxchg_release arch_cmpxchg
  37 #define arch_cmpxchg_relaxed arch_cmpxchg
  38 #else /* arch_cmpxchg_relaxed */
  39
  40 #ifndef arch_cmpxchg_acquire
  41 #define arch_cmpxchg_acquire(...) \
  42         __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
  43 #endif
  44
  45 #ifndef arch_cmpxchg_release
  46 #define arch_cmpxchg_release(...) \
  47         __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
  48 #endif
  49
  50 #ifndef arch_cmpxchg
  51 #define arch_cmpxchg(...) \
  52         __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
  53 #endif
  54
  55 #endif /* arch_cmpxchg_relaxed */
  56
  57 #ifndef arch_cmpxchg64_relaxed
  58 #define arch_cmpxchg64_acquire arch_cmpxchg64
  59 #define arch_cmpxchg64_release arch_cmpxchg64
  60 #define arch_cmpxchg64_relaxed arch_cmpxchg64
  61 #else /* arch_cmpxchg64_relaxed */
  62
  63 #ifndef arch_cmpxchg64_acquire
  64 #define arch_cmpxchg64_acquire(...) \
  65         __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
  66 #endif
  67
  68 #ifndef arch_cmpxchg64_release
  69 #define arch_cmpxchg64_release(...) \
  70         __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
  71 #endif
  72
  73 #ifndef arch_cmpxchg64
  74 #define arch_cmpxchg64(...) \
  75         __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
  76 #endif
  77
  78 #endif /* arch_cmpxchg64_relaxed */
  79
  80 #ifndef arch_try_cmpxchg_relaxed
  81 #ifdef arch_try_cmpxchg
  82 #define arch_try_cmpxchg_acquire arch_try_cmpxchg
  83 #define arch_try_cmpxchg_release arch_try_cmpxchg
  84 #define arch_try_cmpxchg_relaxed arch_try_cmpxchg
  85 #endif /* arch_try_cmpxchg */
  86
  87 #ifndef arch_try_cmpxchg
  88 #define arch_try_cmpxchg(_ptr, _oldp, _new) \
  89 ({ \
  90         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
  91         ___r = arch_cmpxchg((_ptr), ___o, (_new)); \
  92         if (unlikely(___r != ___o)) \
  93                 *___op = ___r; \
  94         likely(___r == ___o); \
  95 })
  96 #endif /* arch_try_cmpxchg */
  97
  98 #ifndef arch_try_cmpxchg_acquire
  99 #define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \
 100 ({ \
 101         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 102         ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \
 103         if (unlikely(___r != ___o)) \
 104                 *___op = ___r; \
 105         likely(___r == ___o); \
 106 })
 107 #endif /* arch_try_cmpxchg_acquire */
 108
 109 #ifndef arch_try_cmpxchg_release
 110 #define arch_try_cmpxchg_release(_ptr, _oldp, _new) \
 111 ({ \
 112         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 113         ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \
 114         if (unlikely(___r != ___o)) \
 115                 *___op = ___r; \
 116         likely(___r == ___o); \
 117 })
 118 #endif /* arch_try_cmpxchg_release */
 119
 120 #ifndef arch_try_cmpxchg_relaxed
 121 #define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
 122 ({ \
 123         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 124         ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \
 125         if (unlikely(___r != ___o)) \
 126                 *___op = ___r; \
 127         likely(___r == ___o); \
 128 })
 129 #endif /* arch_try_cmpxchg_relaxed */
 130
 131 #else /* arch_try_cmpxchg_relaxed */
 132
 133 #ifndef arch_try_cmpxchg_acquire
 134 #define arch_try_cmpxchg_acquire(...) \
 135         __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
 136 #endif
 137
 138 #ifndef arch_try_cmpxchg_release
 139 #define arch_try_cmpxchg_release(...) \
 140         __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
 141 #endif
 142
 143 #ifndef arch_try_cmpxchg
 144 #define arch_try_cmpxchg(...) \
 145         __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
 146 #endif
 147
 148 #endif /* arch_try_cmpxchg_relaxed */
 149
 150 #ifndef arch_try_cmpxchg64_relaxed
 151 #ifdef arch_try_cmpxchg64
 152 #define arch_try_cmpxchg64_acquire arch_try_cmpxchg64
 153 #define arch_try_cmpxchg64_release arch_try_cmpxchg64
 154 #define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64
 155 #endif /* arch_try_cmpxchg64 */
 156
 157 #ifndef arch_try_cmpxchg64
 158 #define arch_try_cmpxchg64(_ptr, _oldp, _new) \
 159 ({ \
 160         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 161         ___r = arch_cmpxchg64((_ptr), ___o, (_new)); \
 162         if (unlikely(___r != ___o)) \
 163                 *___op = ___r; \
 164         likely(___r == ___o); \
 165 })
 166 #endif /* arch_try_cmpxchg64 */
 167
 168 #ifndef arch_try_cmpxchg64_acquire
 169 #define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
 170 ({ \
 171         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 172         ___r = arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \
 173         if (unlikely(___r != ___o)) \
 174                 *___op = ___r; \
 175         likely(___r == ___o); \
 176 })
 177 #endif /* arch_try_cmpxchg64_acquire */
 178
 179 #ifndef arch_try_cmpxchg64_release
 180 #define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \
 181 ({ \
 182         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 183         ___r = arch_cmpxchg64_release((_ptr), ___o, (_new)); \
 184         if (unlikely(___r != ___o)) \
 185                 *___op = ___r; \
 186         likely(___r == ___o); \
 187 })
 188 #endif /* arch_try_cmpxchg64_release */
 189
 190 #ifndef arch_try_cmpxchg64_relaxed
 191 #define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
 192 ({ \
 193         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 194         ___r = arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
 195         if (unlikely(___r != ___o)) \
 196                 *___op = ___r; \
 197         likely(___r == ___o); \
 198 })
 199 #endif /* arch_try_cmpxchg64_relaxed */
 200
 201 #else /* arch_try_cmpxchg64_relaxed */
 202
 203 #ifndef arch_try_cmpxchg64_acquire
 204 #define arch_try_cmpxchg64_acquire(...) \
 205         __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
 206 #endif
 207
 208 #ifndef arch_try_cmpxchg64_release
 209 #define arch_try_cmpxchg64_release(...) \
 210         __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
 211 #endif
 212
 213 #ifndef arch_try_cmpxchg64
 214 #define arch_try_cmpxchg64(...) \
 215         __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
 216 #endif
 217
 218 #endif /* arch_try_cmpxchg64_relaxed */
 219
 220 #ifndef arch_atomic_read_acquire
 221 static __always_inline int
 222 arch_atomic_read_acquire(const atomic_t *v)
 223 {
 224         int ret;
 225
 226         if (__native_word(atomic_t)) {
 227                 ret = smp_load_acquire(&(v)->counter);
 228         } else {
 229                 ret = arch_atomic_read(v);
 230                 __atomic_acquire_fence();
 231         }
 232
 233         return ret;
 234 }
 235 #define arch_atomic_read_acquire arch_atomic_read_acquire
 236 #endif
 237
 238 #ifndef arch_atomic_set_release
 239 static __always_inline void
 240 arch_atomic_set_release(atomic_t *v, int i)
 241 {
 242         if (__native_word(atomic_t)) {
 243                 smp_store_release(&(v)->counter, i);
 244         } else {
 245                 __atomic_release_fence();
 246                 arch_atomic_set(v, i);
 247         }
 248 }
 249 #define arch_atomic_set_release arch_atomic_set_release
 250 #endif
 251
 252 #ifndef arch_atomic_add_return_relaxed
 253 #define arch_atomic_add_return_acquire arch_atomic_add_return
 254 #define arch_atomic_add_return_release arch_atomic_add_return
 255 #define arch_atomic_add_return_relaxed arch_atomic_add_return
 256 #else /* arch_atomic_add_return_relaxed */
 257
 258 #ifndef arch_atomic_add_return_acquire
 259 static __always_inline int
 260 arch_atomic_add_return_acquire(int i, atomic_t *v)
 261 {
 262         int ret = arch_atomic_add_return_relaxed(i, v);
 263         __atomic_acquire_fence();
 264         return ret;
 265 }
 266 #define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
 267 #endif
 268
 269 #ifndef arch_atomic_add_return_release
 270 static __always_inline int
 271 arch_atomic_add_return_release(int i, atomic_t *v)
 272 {
 273         __atomic_release_fence();
 274         return arch_atomic_add_return_relaxed(i, v);
 275 }
 276 #define arch_atomic_add_return_release arch_atomic_add_return_release
 277 #endif
 278
 279 #ifndef arch_atomic_add_return
 280 static __always_inline int
 281 arch_atomic_add_return(int i, atomic_t *v)
 282 {
 283         int ret;
 284         __atomic_pre_full_fence();
 285         ret = arch_atomic_add_return_relaxed(i, v);
 286         __atomic_post_full_fence();
 287         return ret;
 288 }
 289 #define arch_atomic_add_return arch_atomic_add_return
 290 #endif
 291
 292 #endif /* arch_atomic_add_return_relaxed */
 293
 294 #ifndef arch_atomic_fetch_add_relaxed
 295 #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
 296 #define arch_atomic_fetch_add_release arch_atomic_fetch_add
 297 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
 298 #else /* arch_atomic_fetch_add_relaxed */
 299
 300 #ifndef arch_atomic_fetch_add_acquire
 301 static __always_inline int
 302 arch_atomic_fetch_add_acquire(int i, atomic_t *v)
 303 {
 304         int ret = arch_atomic_fetch_add_relaxed(i, v);
 305         __atomic_acquire_fence();
 306         return ret;
 307 }
 308 #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
 309 #endif
 310
 311 #ifndef arch_atomic_fetch_add_release
 312 static __always_inline int
 313 arch_atomic_fetch_add_release(int i, atomic_t *v)
 314 {
 315         __atomic_release_fence();
 316         return arch_atomic_fetch_add_relaxed(i, v);
 317 }
 318 #define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
 319 #endif
 320
 321 #ifndef arch_atomic_fetch_add
 322 static __always_inline int
 323 arch_atomic_fetch_add(int i, atomic_t *v)
 324 {
 325         int ret;
 326         __atomic_pre_full_fence();
 327         ret = arch_atomic_fetch_add_relaxed(i, v);
 328         __atomic_post_full_fence();
 329         return ret;
 330 }
 331 #define arch_atomic_fetch_add arch_atomic_fetch_add
 332 #endif
 333
 334 #endif /* arch_atomic_fetch_add_relaxed */
 335
 336 #ifndef arch_atomic_sub_return_relaxed
 337 #define arch_atomic_sub_return_acquire arch_atomic_sub_return
 338 #define arch_atomic_sub_return_release arch_atomic_sub_return
 339 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return
 340 #else /* arch_atomic_sub_return_relaxed */
 341
 342 #ifndef arch_atomic_sub_return_acquire
 343 static __always_inline int
 344 arch_atomic_sub_return_acquire(int i, atomic_t *v)
 345 {
 346         int ret = arch_atomic_sub_return_relaxed(i, v);
 347         __atomic_acquire_fence();
 348         return ret;
 349 }
 350 #define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
 351 #endif
 352
 353 #ifndef arch_atomic_sub_return_release
 354 static __always_inline int
 355 arch_atomic_sub_return_release(int i, atomic_t *v)
 356 {
 357         __atomic_release_fence();
 358         return arch_atomic_sub_return_relaxed(i, v);
 359 }
 360 #define arch_atomic_sub_return_release arch_atomic_sub_return_release
 361 #endif
 362
 363 #ifndef arch_atomic_sub_return
 364 static __always_inline int
 365 arch_atomic_sub_return(int i, atomic_t *v)
 366 {
 367         int ret;
 368         __atomic_pre_full_fence();
 369         ret = arch_atomic_sub_return_relaxed(i, v);
 370         __atomic_post_full_fence();
 371         return ret;
 372 }
 373 #define arch_atomic_sub_return arch_atomic_sub_return
 374 #endif
 375
 376 #endif /* arch_atomic_sub_return_relaxed */
 377
 378 #ifndef arch_atomic_fetch_sub_relaxed
 379 #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
 380 #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
 381 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
 382 #else /* arch_atomic_fetch_sub_relaxed */
 383
 384 #ifndef arch_atomic_fetch_sub_acquire
 385 static __always_inline int
 386 arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
 387 {
 388         int ret = arch_atomic_fetch_sub_relaxed(i, v);
 389         __atomic_acquire_fence();
 390         return ret;
 391 }
 392 #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
 393 #endif
 394
 395 #ifndef arch_atomic_fetch_sub_release
 396 static __always_inline int
 397 arch_atomic_fetch_sub_release(int i, atomic_t *v)
 398 {
 399         __atomic_release_fence();
 400         return arch_atomic_fetch_sub_relaxed(i, v);
 401 }
 402 #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
 403 #endif
 404
 405 #ifndef arch_atomic_fetch_sub
 406 static __always_inline int
 407 arch_atomic_fetch_sub(int i, atomic_t *v)
 408 {
 409         int ret;
 410         __atomic_pre_full_fence();
 411         ret = arch_atomic_fetch_sub_relaxed(i, v);
 412         __atomic_post_full_fence();
 413         return ret;
 414 }
 415 #define arch_atomic_fetch_sub arch_atomic_fetch_sub
 416 #endif
 417
 418 #endif /* arch_atomic_fetch_sub_relaxed */
 419
 420 #ifndef arch_atomic_inc
 421 static __always_inline void
 422 arch_atomic_inc(atomic_t *v)
 423 {
 424         arch_atomic_add(1, v);
 425 }
 426 #define arch_atomic_inc arch_atomic_inc
 427 #endif
 428
 429 #ifndef arch_atomic_inc_return_relaxed
 430 #ifdef arch_atomic_inc_return
 431 #define arch_atomic_inc_return_acquire arch_atomic_inc_return
 432 #define arch_atomic_inc_return_release arch_atomic_inc_return
 433 #define arch_atomic_inc_return_relaxed arch_atomic_inc_return
 434 #endif /* arch_atomic_inc_return */
 435
 436 #ifndef arch_atomic_inc_return
 437 static __always_inline int
 438 arch_atomic_inc_return(atomic_t *v)
 439 {
 440         return arch_atomic_add_return(1, v);
 441 }
 442 #define arch_atomic_inc_return arch_atomic_inc_return
 443 #endif
 444
 445 #ifndef arch_atomic_inc_return_acquire
 446 static __always_inline int
 447 arch_atomic_inc_return_acquire(atomic_t *v)
 448 {
 449         return arch_atomic_add_return_acquire(1, v);
 450 }
 451 #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
 452 #endif
 453
 454 #ifndef arch_atomic_inc_return_release
 455 static __always_inline int
 456 arch_atomic_inc_return_release(atomic_t *v)
 457 {
 458         return arch_atomic_add_return_release(1, v);
 459 }
 460 #define arch_atomic_inc_return_release arch_atomic_inc_return_release
 461 #endif
 462
 463 #ifndef arch_atomic_inc_return_relaxed
 464 static __always_inline int
 465 arch_atomic_inc_return_relaxed(atomic_t *v)
 466 {
 467         return arch_atomic_add_return_relaxed(1, v);
 468 }
 469 #define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
 470 #endif
 471
 472 #else /* arch_atomic_inc_return_relaxed */
 473
 474 #ifndef arch_atomic_inc_return_acquire
 475 static __always_inline int
 476 arch_atomic_inc_return_acquire(atomic_t *v)
 477 {
 478         int ret = arch_atomic_inc_return_relaxed(v);
 479         __atomic_acquire_fence();
 480         return ret;
 481 }
 482 #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
 483 #endif
 484
 485 #ifndef arch_atomic_inc_return_release
 486 static __always_inline int
 487 arch_atomic_inc_return_release(atomic_t *v)
 488 {
 489         __atomic_release_fence();
 490         return arch_atomic_inc_return_relaxed(v);
 491 }
 492 #define arch_atomic_inc_return_release arch_atomic_inc_return_release
 493 #endif
 494
 495 #ifndef arch_atomic_inc_return
 496 static __always_inline int
 497 arch_atomic_inc_return(atomic_t *v)
 498 {
 499         int ret;
 500         __atomic_pre_full_fence();
 501         ret = arch_atomic_inc_return_relaxed(v);
 502         __atomic_post_full_fence();
 503         return ret;
 504 }
 505 #define arch_atomic_inc_return arch_atomic_inc_return
 506 #endif
 507
 508 #endif /* arch_atomic_inc_return_relaxed */
 509
 510 #ifndef arch_atomic_fetch_inc_relaxed
 511 #ifdef arch_atomic_fetch_inc
 512 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
 513 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
 514 #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
 515 #endif /* arch_atomic_fetch_inc */
 516
 517 #ifndef arch_atomic_fetch_inc
 518 static __always_inline int
 519 arch_atomic_fetch_inc(atomic_t *v)
 520 {
 521         return arch_atomic_fetch_add(1, v);
 522 }
 523 #define arch_atomic_fetch_inc arch_atomic_fetch_inc
 524 #endif
 525
 526 #ifndef arch_atomic_fetch_inc_acquire
 527 static __always_inline int
 528 arch_atomic_fetch_inc_acquire(atomic_t *v)
 529 {
 530         return arch_atomic_fetch_add_acquire(1, v);
 531 }
 532 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
 533 #endif
 534
 535 #ifndef arch_atomic_fetch_inc_release
 536 static __always_inline int
 537 arch_atomic_fetch_inc_release(atomic_t *v)
 538 {
 539         return arch_atomic_fetch_add_release(1, v);
 540 }
 541 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
 542 #endif
 543
 544 #ifndef arch_atomic_fetch_inc_relaxed
 545 static __always_inline int
 546 arch_atomic_fetch_inc_relaxed(atomic_t *v)
 547 {
 548         return arch_atomic_fetch_add_relaxed(1, v);
 549 }
 550 #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
 551 #endif
 552
 553 #else /* arch_atomic_fetch_inc_relaxed */
 554
 555 #ifndef arch_atomic_fetch_inc_acquire
 556 static __always_inline int
 557 arch_atomic_fetch_inc_acquire(atomic_t *v)
 558 {
 559         int ret = arch_atomic_fetch_inc_relaxed(v);
 560         __atomic_acquire_fence();
 561         return ret;
 562 }
 563 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
 564 #endif
 565
 566 #ifndef arch_atomic_fetch_inc_release
 567 static __always_inline int
 568 arch_atomic_fetch_inc_release(atomic_t *v)
 569 {
 570         __atomic_release_fence();
 571         return arch_atomic_fetch_inc_relaxed(v);
 572 }
 573 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
 574 #endif
 575
 576 #ifndef arch_atomic_fetch_inc
 577 static __always_inline int
 578 arch_atomic_fetch_inc(atomic_t *v)
 579 {
 580         int ret;
 581         __atomic_pre_full_fence();
 582         ret = arch_atomic_fetch_inc_relaxed(v);
 583         __atomic_post_full_fence();
 584         return ret;
 585 }
 586 #define arch_atomic_fetch_inc arch_atomic_fetch_inc
 587 #endif
 588
 589 #endif /* arch_atomic_fetch_inc_relaxed */
 590
 591 #ifndef arch_atomic_dec
 592 static __always_inline void
 593 arch_atomic_dec(atomic_t *v)
 594 {
 595         arch_atomic_sub(1, v);
 596 }
 597 #define arch_atomic_dec arch_atomic_dec
 598 #endif
 599
 600 #ifndef arch_atomic_dec_return_relaxed
 601 #ifdef arch_atomic_dec_return
 602 #define arch_atomic_dec_return_acquire arch_atomic_dec_return
 603 #define arch_atomic_dec_return_release arch_atomic_dec_return
 604 #define arch_atomic_dec_return_relaxed arch_atomic_dec_return
 605 #endif /* arch_atomic_dec_return */
 606
 607 #ifndef arch_atomic_dec_return
 608 static __always_inline int
 609 arch_atomic_dec_return(atomic_t *v)
 610 {
 611         return arch_atomic_sub_return(1, v);
 612 }
 613 #define arch_atomic_dec_return arch_atomic_dec_return
 614 #endif
 615
 616 #ifndef arch_atomic_dec_return_acquire
 617 static __always_inline int
 618 arch_atomic_dec_return_acquire(atomic_t *v)
 619 {
 620         return arch_atomic_sub_return_acquire(1, v);
 621 }
 622 #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
 623 #endif
 624
 625 #ifndef arch_atomic_dec_return_release
 626 static __always_inline int
 627 arch_atomic_dec_return_release(atomic_t *v)
 628 {
 629         return arch_atomic_sub_return_release(1, v);
 630 }
 631 #define arch_atomic_dec_return_release arch_atomic_dec_return_release
 632 #endif
 633
 634 #ifndef arch_atomic_dec_return_relaxed
 635 static __always_inline int
 636 arch_atomic_dec_return_relaxed(atomic_t *v)
 637 {
 638         return arch_atomic_sub_return_relaxed(1, v);
 639 }
 640 #define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
 641 #endif
 642
 643 #else /* arch_atomic_dec_return_relaxed */
 644
 645 #ifndef arch_atomic_dec_return_acquire
 646 static __always_inline int
 647 arch_atomic_dec_return_acquire(atomic_t *v)
 648 {
 649         int ret = arch_atomic_dec_return_relaxed(v);
 650         __atomic_acquire_fence();
 651         return ret;
 652 }
 653 #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
 654 #endif
 655
 656 #ifndef arch_atomic_dec_return_release
 657 static __always_inline int
 658 arch_atomic_dec_return_release(atomic_t *v)
 659 {
 660         __atomic_release_fence();
 661         return arch_atomic_dec_return_relaxed(v);
 662 }
 663 #define arch_atomic_dec_return_release arch_atomic_dec_return_release
 664 #endif
 665
 666 #ifndef arch_atomic_dec_return
 667 static __always_inline int
 668 arch_atomic_dec_return(atomic_t *v)
 669 {
 670         int ret;
 671         __atomic_pre_full_fence();
 672         ret = arch_atomic_dec_return_relaxed(v);
 673         __atomic_post_full_fence();
 674         return ret;
 675 }
 676 #define arch_atomic_dec_return arch_atomic_dec_return
 677 #endif
 678
 679 #endif /* arch_atomic_dec_return_relaxed */
 680
 681 #ifndef arch_atomic_fetch_dec_relaxed
 682 #ifdef arch_atomic_fetch_dec
 683 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
 684 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
 685 #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
 686 #endif /* arch_atomic_fetch_dec */
 687
 688 #ifndef arch_atomic_fetch_dec
 689 static __always_inline int
 690 arch_atomic_fetch_dec(atomic_t *v)
 691 {
 692         return arch_atomic_fetch_sub(1, v);
 693 }
 694 #define arch_atomic_fetch_dec arch_atomic_fetch_dec
 695 #endif
 696
 697 #ifndef arch_atomic_fetch_dec_acquire
 698 static __always_inline int
 699 arch_atomic_fetch_dec_acquire(atomic_t *v)
 700 {
 701         return arch_atomic_fetch_sub_acquire(1, v);
 702 }
 703 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
 704 #endif
 705
 706 #ifndef arch_atomic_fetch_dec_release
 707 static __always_inline int
 708 arch_atomic_fetch_dec_release(atomic_t *v)
 709 {
 710         return arch_atomic_fetch_sub_release(1, v);
 711 }
 712 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
 713 #endif
 714
 715 #ifndef arch_atomic_fetch_dec_relaxed
 716 static __always_inline int
 717 arch_atomic_fetch_dec_relaxed(atomic_t *v)
 718 {
 719         return arch_atomic_fetch_sub_relaxed(1, v);
 720 }
 721 #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
 722 #endif
 723
 724 #else /* arch_atomic_fetch_dec_relaxed */
 725
 726 #ifndef arch_atomic_fetch_dec_acquire
 727 static __always_inline int
 728 arch_atomic_fetch_dec_acquire(atomic_t *v)
 729 {
 730         int ret = arch_atomic_fetch_dec_relaxed(v);
 731         __atomic_acquire_fence();
 732         return ret;
 733 }
 734 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
 735 #endif
 736
 737 #ifndef arch_atomic_fetch_dec_release
 738 static __always_inline int
 739 arch_atomic_fetch_dec_release(atomic_t *v)
 740 {
 741         __atomic_release_fence();
 742         return arch_atomic_fetch_dec_relaxed(v);
 743 }
 744 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
 745 #endif
 746
 747 #ifndef arch_atomic_fetch_dec
 748 static __always_inline int
 749 arch_atomic_fetch_dec(atomic_t *v)
 750 {
 751         int ret;
 752         __atomic_pre_full_fence();
 753         ret = arch_atomic_fetch_dec_relaxed(v);
 754         __atomic_post_full_fence();
 755         return ret;
 756 }
 757 #define arch_atomic_fetch_dec arch_atomic_fetch_dec
 758 #endif
 759
 760 #endif /* arch_atomic_fetch_dec_relaxed */
 761
 762 #ifndef arch_atomic_fetch_and_relaxed
 763 #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
 764 #define arch_atomic_fetch_and_release arch_atomic_fetch_and
 765 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
 766 #else /* arch_atomic_fetch_and_relaxed */
 767
 768 #ifndef arch_atomic_fetch_and_acquire
 769 static __always_inline int
 770 arch_atomic_fetch_and_acquire(int i, atomic_t *v)
 771 {
 772         int ret = arch_atomic_fetch_and_relaxed(i, v);
 773         __atomic_acquire_fence();
 774         return ret;
 775 }
 776 #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
 777 #endif
 778
 779 #ifndef arch_atomic_fetch_and_release
 780 static __always_inline int
 781 arch_atomic_fetch_and_release(int i, atomic_t *v)
 782 {
 783         __atomic_release_fence();
 784         return arch_atomic_fetch_and_relaxed(i, v);
 785 }
 786 #define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
 787 #endif
 788
 789 #ifndef arch_atomic_fetch_and
 790 static __always_inline int
 791 arch_atomic_fetch_and(int i, atomic_t *v)
 792 {
 793         int ret;
 794         __atomic_pre_full_fence();
 795         ret = arch_atomic_fetch_and_relaxed(i, v);
 796         __atomic_post_full_fence();
 797         return ret;
 798 }
 799 #define arch_atomic_fetch_and arch_atomic_fetch_and
 800 #endif
 801
 802 #endif /* arch_atomic_fetch_and_relaxed */
 803
 804 #ifndef arch_atomic_andnot
 805 static __always_inline void
 806 arch_atomic_andnot(int i, atomic_t *v)
 807 {
 808         arch_atomic_and(~i, v);
 809 }
 810 #define arch_atomic_andnot arch_atomic_andnot
 811 #endif
 812
 813 #ifndef arch_atomic_fetch_andnot_relaxed
 814 #ifdef arch_atomic_fetch_andnot
 815 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
 816 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
 817 #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
 818 #endif /* arch_atomic_fetch_andnot */
 819
 820 #ifndef arch_atomic_fetch_andnot
 821 static __always_inline int
 822 arch_atomic_fetch_andnot(int i, atomic_t *v)
 823 {
 824         return arch_atomic_fetch_and(~i, v);
 825 }
 826 #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
 827 #endif
 828
 829 #ifndef arch_atomic_fetch_andnot_acquire
 830 static __always_inline int
 831 arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
 832 {
 833         return arch_atomic_fetch_and_acquire(~i, v);
 834 }
 835 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
 836 #endif
 837
 838 #ifndef arch_atomic_fetch_andnot_release
 839 static __always_inline int
 840 arch_atomic_fetch_andnot_release(int i, atomic_t *v)
 841 {
 842         return arch_atomic_fetch_and_release(~i, v);
 843 }
 844 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
 845 #endif
 846
 847 #ifndef arch_atomic_fetch_andnot_relaxed
 848 static __always_inline int
 849 arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
 850 {
 851         return arch_atomic_fetch_and_relaxed(~i, v);
 852 }
 853 #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
 854 #endif
 855
 856 #else /* arch_atomic_fetch_andnot_relaxed */
 857
 858 #ifndef arch_atomic_fetch_andnot_acquire
 859 static __always_inline int
 860 arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
 861 {
 862         int ret = arch_atomic_fetch_andnot_relaxed(i, v);
 863         __atomic_acquire_fence();
 864         return ret;
 865 }
 866 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
 867 #endif
 868
 869 #ifndef arch_atomic_fetch_andnot_release
 870 static __always_inline int
 871 arch_atomic_fetch_andnot_release(int i, atomic_t *v)
 872 {
 873         __atomic_release_fence();
 874         return arch_atomic_fetch_andnot_relaxed(i, v);
 875 }
 876 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
 877 #endif
 878
 879 #ifndef arch_atomic_fetch_andnot
 880 static __always_inline int
 881 arch_atomic_fetch_andnot(int i, atomic_t *v)
 882 {
 883         int ret;
 884         __atomic_pre_full_fence();
 885         ret = arch_atomic_fetch_andnot_relaxed(i, v);
 886         __atomic_post_full_fence();
 887         return ret;
 888 }
 889 #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
 890 #endif
 891
 892 #endif /* arch_atomic_fetch_andnot_relaxed */
 893
 894 #ifndef arch_atomic_fetch_or_relaxed
 895 #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
 896 #define arch_atomic_fetch_or_release arch_atomic_fetch_or
 897 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
 898 #else /* arch_atomic_fetch_or_relaxed */
 899
 900 #ifndef arch_atomic_fetch_or_acquire
 901 static __always_inline int
 902 arch_atomic_fetch_or_acquire(int i, atomic_t *v)
 903 {
 904         int ret = arch_atomic_fetch_or_relaxed(i, v);
 905         __atomic_acquire_fence();
 906         return ret;
 907 }
 908 #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
 909 #endif
 910
 911 #ifndef arch_atomic_fetch_or_release
 912 static __always_inline int
 913 arch_atomic_fetch_or_release(int i, atomic_t *v)
 914 {
 915         __atomic_release_fence();
 916         return arch_atomic_fetch_or_relaxed(i, v);
 917 }
 918 #define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
 919 #endif
 920
 921 #ifndef arch_atomic_fetch_or
 922 static __always_inline int
 923 arch_atomic_fetch_or(int i, atomic_t *v)
 924 {
 925         int ret;
 926         __atomic_pre_full_fence();
 927         ret = arch_atomic_fetch_or_relaxed(i, v);
 928         __atomic_post_full_fence();
 929         return ret;
 930 }
 931 #define arch_atomic_fetch_or arch_atomic_fetch_or
 932 #endif
 933
 934 #endif /* arch_atomic_fetch_or_relaxed */
 935
 936 #ifndef arch_atomic_fetch_xor_relaxed
 937 #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
 938 #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
 939 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
 940 #else /* arch_atomic_fetch_xor_relaxed */
 941
 942 #ifndef arch_atomic_fetch_xor_acquire
 943 static __always_inline int
 944 arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
 945 {
 946         int ret = arch_atomic_fetch_xor_relaxed(i, v);
 947         __atomic_acquire_fence();
 948         return ret;
 949 }
 950 #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
 951 #endif
 952
 953 #ifndef arch_atomic_fetch_xor_release
 954 static __always_inline int
 955 arch_atomic_fetch_xor_release(int i, atomic_t *v)
 956 {
 957         __atomic_release_fence();
 958         return arch_atomic_fetch_xor_relaxed(i, v);
 959 }
 960 #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
 961 #endif
 962
 963 #ifndef arch_atomic_fetch_xor
 964 static __always_inline int
 965 arch_atomic_fetch_xor(int i, atomic_t *v)
 966 {
 967         int ret;
 968         __atomic_pre_full_fence();
 969         ret = arch_atomic_fetch_xor_relaxed(i, v);
 970         __atomic_post_full_fence();
 971         return ret;
 972 }
 973 #define arch_atomic_fetch_xor arch_atomic_fetch_xor
 974 #endif
 975
 976 #endif /* arch_atomic_fetch_xor_relaxed */
 977
 978 #ifndef arch_atomic_xchg_relaxed
 979 #define arch_atomic_xchg_acquire arch_atomic_xchg
 980 #define arch_atomic_xchg_release arch_atomic_xchg
 981 #define arch_atomic_xchg_relaxed arch_atomic_xchg
 982 #else /* arch_atomic_xchg_relaxed */
 983
 984 #ifndef arch_atomic_xchg_acquire
 985 static __always_inline int
 986 arch_atomic_xchg_acquire(atomic_t *v, int i)
 987 {
 988         int ret = arch_atomic_xchg_relaxed(v, i);
 989         __atomic_acquire_fence();
 990         return ret;
 991 }
 992 #define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
 993 #endif
 994
 995 #ifndef arch_atomic_xchg_release
 996 static __always_inline int
 997 arch_atomic_xchg_release(atomic_t *v, int i)
 998 {
 999         __atomic_release_fence();
1000         return arch_atomic_xchg_relaxed(v, i);
1001 }
1002 #define arch_atomic_xchg_release arch_atomic_xchg_release
1003 #endif
1004
1005 #ifndef arch_atomic_xchg
1006 static __always_inline int
1007 arch_atomic_xchg(atomic_t *v, int i)
1008 {
1009         int ret;
1010         __atomic_pre_full_fence();
1011         ret = arch_atomic_xchg_relaxed(v, i);
1012         __atomic_post_full_fence();
1013         return ret;
1014 }
1015 #define arch_atomic_xchg arch_atomic_xchg
1016 #endif
1017
1018 #endif /* arch_atomic_xchg_relaxed */
1019
1020 #ifndef arch_atomic_cmpxchg_relaxed
1021 #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
1022 #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
1023 #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
1024 #else /* arch_atomic_cmpxchg_relaxed */
1025
1026 #ifndef arch_atomic_cmpxchg_acquire
1027 static __always_inline int
1028 arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
1029 {
1030         int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
1031         __atomic_acquire_fence();
1032         return ret;
1033 }
1034 #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
1035 #endif
1036
1037 #ifndef arch_atomic_cmpxchg_release
1038 static __always_inline int
1039 arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
1040 {
1041         __atomic_release_fence();
1042         return arch_atomic_cmpxchg_relaxed(v, old, new);
1043 }
1044 #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
1045 #endif
1046
1047 #ifndef arch_atomic_cmpxchg
1048 static __always_inline int
1049 arch_atomic_cmpxchg(atomic_t *v, int old, int new)
1050 {
1051         int ret;
1052         __atomic_pre_full_fence();
1053         ret = arch_atomic_cmpxchg_relaxed(v, old, new);
1054         __atomic_post_full_fence();
1055         return ret;
1056 }
1057 #define arch_atomic_cmpxchg arch_atomic_cmpxchg
1058 #endif
1059
1060 #endif /* arch_atomic_cmpxchg_relaxed */
1061
1062 #ifndef arch_atomic_try_cmpxchg_relaxed
1063 #ifdef arch_atomic_try_cmpxchg
1064 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
1065 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
1066 #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
1067 #endif /* arch_atomic_try_cmpxchg */
1068
1069 #ifndef arch_atomic_try_cmpxchg
1070 static __always_inline bool
1071 arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1072 {
1073         int r, o = *old;
1074         r = arch_atomic_cmpxchg(v, o, new);
1075         if (unlikely(r != o))
1076                 *old = r;
1077         return likely(r == o);
1078 }
1079 #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1080 #endif
1081
1082 #ifndef arch_atomic_try_cmpxchg_acquire
1083 static __always_inline bool
1084 arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1085 {
1086         int r, o = *old;
1087         r = arch_atomic_cmpxchg_acquire(v, o, new);
1088         if (unlikely(r != o))
1089                 *old = r;
1090         return likely(r == o);
1091 }
1092 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1093 #endif
1094
1095 #ifndef arch_atomic_try_cmpxchg_release
1096 static __always_inline bool
1097 arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1098 {
1099         int r, o = *old;
1100         r = arch_atomic_cmpxchg_release(v, o, new);
1101         if (unlikely(r != o))
1102                 *old = r;
1103         return likely(r == o);
1104 }
1105 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1106 #endif
1107
1108 #ifndef arch_atomic_try_cmpxchg_relaxed
1109 static __always_inline bool
1110 arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1111 {
1112         int r, o = *old;
1113         r = arch_atomic_cmpxchg_relaxed(v, o, new);
1114         if (unlikely(r != o))
1115                 *old = r;
1116         return likely(r == o);
1117 }
1118 #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
1119 #endif
1120
1121 #else /* arch_atomic_try_cmpxchg_relaxed */
1122
1123 #ifndef arch_atomic_try_cmpxchg_acquire
1124 static __always_inline bool
1125 arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1126 {
1127         bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1128         __atomic_acquire_fence();
1129         return ret;
1130 }
1131 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1132 #endif
1133
1134 #ifndef arch_atomic_try_cmpxchg_release
1135 static __always_inline bool
1136 arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1137 {
1138         __atomic_release_fence();
1139         return arch_atomic_try_cmpxchg_relaxed(v, old, new);
1140 }
1141 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1142 #endif
1143
1144 #ifndef arch_atomic_try_cmpxchg
1145 static __always_inline bool
1146 arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1147 {
1148         bool ret;
1149         __atomic_pre_full_fence();
1150         ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1151         __atomic_post_full_fence();
1152         return ret;
1153 }
1154 #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1155 #endif
1156
1157 #endif /* arch_atomic_try_cmpxchg_relaxed */
1158
1159 #ifndef arch_atomic_sub_and_test
1160 /**
1161  * arch_atomic_sub_and_test - subtract value from variable and test result
1162  * @i: integer value to subtract
1163  * @v: pointer of type atomic_t
1164  *
1165  * Atomically subtracts @i from @v and returns
1166  * true if the result is zero, or false for all
1167  * other cases.
1168  */
1169 static __always_inline bool
1170 arch_atomic_sub_and_test(int i, atomic_t *v)
1171 {
1172         return arch_atomic_sub_return(i, v) == 0;
1173 }
1174 #define arch_atomic_sub_and_test arch_atomic_sub_and_test
1175 #endif
1176
1177 #ifndef arch_atomic_dec_and_test
1178 /**
1179  * arch_atomic_dec_and_test - decrement and test
1180  * @v: pointer of type atomic_t
1181  *
1182  * Atomically decrements @v by 1 and
1183  * returns true if the result is 0, or false for all other
1184  * cases.
1185  */
1186 static __always_inline bool
1187 arch_atomic_dec_and_test(atomic_t *v)
1188 {
1189         return arch_atomic_dec_return(v) == 0;
1190 }
1191 #define arch_atomic_dec_and_test arch_atomic_dec_and_test
1192 #endif
1193
1194 #ifndef arch_atomic_inc_and_test
1195 /**
1196  * arch_atomic_inc_and_test - increment and test
1197  * @v: pointer of type atomic_t
1198  *
1199  * Atomically increments @v by 1
1200  * and returns true if the result is zero, or false for all
1201  * other cases.
1202  */
1203 static __always_inline bool
1204 arch_atomic_inc_and_test(atomic_t *v)
1205 {
1206         return arch_atomic_inc_return(v) == 0;
1207 }
1208 #define arch_atomic_inc_and_test arch_atomic_inc_and_test
1209 #endif
1210
1211 #ifndef arch_atomic_add_negative
1212 /**
1213  * arch_atomic_add_negative - add and test if negative
1214  * @i: integer value to add
1215  * @v: pointer of type atomic_t
1216  *
1217  * Atomically adds @i to @v and returns true
1218  * if the result is negative, or false when
1219  * result is greater than or equal to zero.
1220  */
1221 static __always_inline bool
1222 arch_atomic_add_negative(int i, atomic_t *v)
1223 {
1224         return arch_atomic_add_return(i, v) < 0;
1225 }
1226 #define arch_atomic_add_negative arch_atomic_add_negative
1227 #endif
1228
1229 #ifndef arch_atomic_fetch_add_unless
1230 /**
1231  * arch_atomic_fetch_add_unless - add unless the number is already a given value
1232  * @v: pointer of type atomic_t
1233  * @a: the amount to add to v...
1234  * @u: ...unless v is equal to u.
1235  *
1236  * Atomically adds @a to @v, so long as @v was not already @u.
1237  * Returns original value of @v
1238  */
1239 static __always_inline int
1240 arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
1241 {
1242         int c = arch_atomic_read(v);
1243
1244         do {
1245                 if (unlikely(c == u))
1246                         break;
1247         } while (!arch_atomic_try_cmpxchg(v, &c, c + a));
1248
1249         return c;
1250 }
1251 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
1252 #endif
1253
1254 #ifndef arch_atomic_add_unless
1255 /**
1256  * arch_atomic_add_unless - add unless the number is already a given value
1257  * @v: pointer of type atomic_t
1258  * @a: the amount to add to v...
1259  * @u: ...unless v is equal to u.
1260  *
1261  * Atomically adds @a to @v, if @v was not already @u.
1262  * Returns true if the addition was done.
1263  */
1264 static __always_inline bool
1265 arch_atomic_add_unless(atomic_t *v, int a, int u)
1266 {
1267         return arch_atomic_fetch_add_unless(v, a, u) != u;
1268 }
1269 #define arch_atomic_add_unless arch_atomic_add_unless
1270 #endif
1271
1272 #ifndef arch_atomic_inc_not_zero
1273 /**
1274  * arch_atomic_inc_not_zero - increment unless the number is zero
1275  * @v: pointer of type atomic_t
1276  *
1277  * Atomically increments @v by 1, if @v is non-zero.
1278  * Returns true if the increment was done.
1279  */
1280 static __always_inline bool
1281 arch_atomic_inc_not_zero(atomic_t *v)
1282 {
1283         return arch_atomic_add_unless(v, 1, 0);
1284 }
1285 #define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
1286 #endif
1287
1288 #ifndef arch_atomic_inc_unless_negative
1289 static __always_inline bool
1290 arch_atomic_inc_unless_negative(atomic_t *v)
1291 {
1292         int c = arch_atomic_read(v);
1293
1294         do {
1295                 if (unlikely(c < 0))
1296                         return false;
1297         } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
1298
1299         return true;
1300 }
1301 #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
1302 #endif
1303
1304 #ifndef arch_atomic_dec_unless_positive
1305 static __always_inline bool
1306 arch_atomic_dec_unless_positive(atomic_t *v)
1307 {
1308         int c = arch_atomic_read(v);
1309
1310         do {
1311                 if (unlikely(c > 0))
1312                         return false;
1313         } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
1314
1315         return true;
1316 }
1317 #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
1318 #endif
1319
1320 #ifndef arch_atomic_dec_if_positive
1321 static __always_inline int
1322 arch_atomic_dec_if_positive(atomic_t *v)
1323 {
1324         int dec, c = arch_atomic_read(v);
1325
1326         do {
1327                 dec = c - 1;
1328                 if (unlikely(dec < 0))
1329                         break;
1330         } while (!arch_atomic_try_cmpxchg(v, &c, dec));
1331
1332         return dec;
1333 }
1334 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
1335 #endif
1336
1337 #ifdef CONFIG_GENERIC_ATOMIC64
1338 #include <asm-generic/atomic64.h>
1339 #endif
1340
1341 #ifndef arch_atomic64_read_acquire
1342 static __always_inline s64
1343 arch_atomic64_read_acquire(const atomic64_t *v)
1344 {
1345         s64 ret;
1346
1347         if (__native_word(atomic64_t)) {
1348                 ret = smp_load_acquire(&(v)->counter);
1349         } else {
1350                 ret = arch_atomic64_read(v);
1351                 __atomic_acquire_fence();
1352         }
1353
1354         return ret;
1355 }
1356 #define arch_atomic64_read_acquire arch_atomic64_read_acquire
1357 #endif
1358
1359 #ifndef arch_atomic64_set_release
1360 static __always_inline void
1361 arch_atomic64_set_release(atomic64_t *v, s64 i)
1362 {
1363         if (__native_word(atomic64_t)) {
1364                 smp_store_release(&(v)->counter, i);
1365         } else {
1366                 __atomic_release_fence();
1367                 arch_atomic64_set(v, i);
1368         }
1369 }
1370 #define arch_atomic64_set_release arch_atomic64_set_release
1371 #endif
1372
1373 #ifndef arch_atomic64_add_return_relaxed
1374 #define arch_atomic64_add_return_acquire arch_atomic64_add_return
1375 #define arch_atomic64_add_return_release arch_atomic64_add_return
1376 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return
1377 #else /* arch_atomic64_add_return_relaxed */
1378
1379 #ifndef arch_atomic64_add_return_acquire
1380 static __always_inline s64
1381 arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
1382 {
1383         s64 ret = arch_atomic64_add_return_relaxed(i, v);
1384         __atomic_acquire_fence();
1385         return ret;
1386 }
1387 #define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
1388 #endif
1389
1390 #ifndef arch_atomic64_add_return_release
1391 static __always_inline s64
1392 arch_atomic64_add_return_release(s64 i, atomic64_t *v)
1393 {
1394         __atomic_release_fence();
1395         return arch_atomic64_add_return_relaxed(i, v);
1396 }
1397 #define arch_atomic64_add_return_release arch_atomic64_add_return_release
1398 #endif
1399
1400 #ifndef arch_atomic64_add_return
1401 static __always_inline s64
1402 arch_atomic64_add_return(s64 i, atomic64_t *v)
1403 {
1404         s64 ret;
1405         __atomic_pre_full_fence();
1406         ret = arch_atomic64_add_return_relaxed(i, v);
1407         __atomic_post_full_fence();
1408         return ret;
1409 }
1410 #define arch_atomic64_add_return arch_atomic64_add_return
1411 #endif
1412
1413 #endif /* arch_atomic64_add_return_relaxed */
1414
1415 #ifndef arch_atomic64_fetch_add_relaxed
1416 #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
1417 #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
1418 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
1419 #else /* arch_atomic64_fetch_add_relaxed */
1420
1421 #ifndef arch_atomic64_fetch_add_acquire
1422 static __always_inline s64
1423 arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1424 {
1425         s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
1426         __atomic_acquire_fence();
1427         return ret;
1428 }
1429 #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
1430 #endif
1431
1432 #ifndef arch_atomic64_fetch_add_release
1433 static __always_inline s64
1434 arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
1435 {
1436         __atomic_release_fence();
1437         return arch_atomic64_fetch_add_relaxed(i, v);
1438 }
1439 #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
1440 #endif
1441
1442 #ifndef arch_atomic64_fetch_add
1443 static __always_inline s64
1444 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
1445 {
1446         s64 ret;
1447         __atomic_pre_full_fence();
1448         ret = arch_atomic64_fetch_add_relaxed(i, v);
1449         __atomic_post_full_fence();
1450         return ret;
1451 }
1452 #define arch_atomic64_fetch_add arch_atomic64_fetch_add
1453 #endif
1454
1455 #endif /* arch_atomic64_fetch_add_relaxed */
1456
1457 #ifndef arch_atomic64_sub_return_relaxed
1458 #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
1459 #define arch_atomic64_sub_return_release arch_atomic64_sub_return
1460 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
1461 #else /* arch_atomic64_sub_return_relaxed */
1462
1463 #ifndef arch_atomic64_sub_return_acquire
1464 static __always_inline s64
1465 arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1466 {
1467         s64 ret = arch_atomic64_sub_return_relaxed(i, v);
1468         __atomic_acquire_fence();
1469         return ret;
1470 }
1471 #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
1472 #endif
1473
1474 #ifndef arch_atomic64_sub_return_release
1475 static __always_inline s64
1476 arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
1477 {
1478         __atomic_release_fence();
1479         return arch_atomic64_sub_return_relaxed(i, v);
1480 }
1481 #define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
1482 #endif
1483
1484 #ifndef arch_atomic64_sub_return
1485 static __always_inline s64
1486 arch_atomic64_sub_return(s64 i, atomic64_t *v)
1487 {
1488         s64 ret;
1489         __atomic_pre_full_fence();
1490         ret = arch_atomic64_sub_return_relaxed(i, v);
1491         __atomic_post_full_fence();
1492         return ret;
1493 }
1494 #define arch_atomic64_sub_return arch_atomic64_sub_return
1495 #endif
1496
1497 #endif /* arch_atomic64_sub_return_relaxed */
1498
1499 #ifndef arch_atomic64_fetch_sub_relaxed
1500 #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
1501 #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
1502 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
1503 #else /* arch_atomic64_fetch_sub_relaxed */
1504
1505 #ifndef arch_atomic64_fetch_sub_acquire
1506 static __always_inline s64
1507 arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1508 {
1509         s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1510         __atomic_acquire_fence();
1511         return ret;
1512 }
1513 #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
1514 #endif
1515
1516 #ifndef arch_atomic64_fetch_sub_release
1517 static __always_inline s64
1518 arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1519 {
1520         __atomic_release_fence();
1521         return arch_atomic64_fetch_sub_relaxed(i, v);
1522 }
1523 #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
1524 #endif
1525
1526 #ifndef arch_atomic64_fetch_sub
1527 static __always_inline s64
1528 arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
1529 {
1530         s64 ret;
1531         __atomic_pre_full_fence();
1532         ret = arch_atomic64_fetch_sub_relaxed(i, v);
1533         __atomic_post_full_fence();
1534         return ret;
1535 }
1536 #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
1537 #endif
1538
1539 #endif /* arch_atomic64_fetch_sub_relaxed */
1540
1541 #ifndef arch_atomic64_inc
1542 static __always_inline void
1543 arch_atomic64_inc(atomic64_t *v)
1544 {
1545         arch_atomic64_add(1, v);
1546 }
1547 #define arch_atomic64_inc arch_atomic64_inc
1548 #endif
1549
1550 #ifndef arch_atomic64_inc_return_relaxed
1551 #ifdef arch_atomic64_inc_return
1552 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
1553 #define arch_atomic64_inc_return_release arch_atomic64_inc_return
1554 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
1555 #endif /* arch_atomic64_inc_return */
1556
1557 #ifndef arch_atomic64_inc_return
1558 static __always_inline s64
1559 arch_atomic64_inc_return(atomic64_t *v)
1560 {
1561         return arch_atomic64_add_return(1, v);
1562 }
1563 #define arch_atomic64_inc_return arch_atomic64_inc_return
1564 #endif
1565
1566 #ifndef arch_atomic64_inc_return_acquire
1567 static __always_inline s64
1568 arch_atomic64_inc_return_acquire(atomic64_t *v)
1569 {
1570         return arch_atomic64_add_return_acquire(1, v);
1571 }
1572 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1573 #endif
1574
1575 #ifndef arch_atomic64_inc_return_release
1576 static __always_inline s64
1577 arch_atomic64_inc_return_release(atomic64_t *v)
1578 {
1579         return arch_atomic64_add_return_release(1, v);
1580 }
1581 #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1582 #endif
1583
1584 #ifndef arch_atomic64_inc_return_relaxed
1585 static __always_inline s64
1586 arch_atomic64_inc_return_relaxed(atomic64_t *v)
1587 {
1588         return arch_atomic64_add_return_relaxed(1, v);
1589 }
1590 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
1591 #endif
1592
1593 #else /* arch_atomic64_inc_return_relaxed */
1594
1595 #ifndef arch_atomic64_inc_return_acquire
1596 static __always_inline s64
1597 arch_atomic64_inc_return_acquire(atomic64_t *v)
1598 {
1599         s64 ret = arch_atomic64_inc_return_relaxed(v);
1600         __atomic_acquire_fence();
1601         return ret;
1602 }
1603 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1604 #endif
1605
1606 #ifndef arch_atomic64_inc_return_release
1607 static __always_inline s64
1608 arch_atomic64_inc_return_release(atomic64_t *v)
1609 {
1610         __atomic_release_fence();
1611         return arch_atomic64_inc_return_relaxed(v);
1612 }
1613 #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1614 #endif
1615
1616 #ifndef arch_atomic64_inc_return
1617 static __always_inline s64
1618 arch_atomic64_inc_return(atomic64_t *v)
1619 {
1620         s64 ret;
1621         __atomic_pre_full_fence();
1622         ret = arch_atomic64_inc_return_relaxed(v);
1623         __atomic_post_full_fence();
1624         return ret;
1625 }
1626 #define arch_atomic64_inc_return arch_atomic64_inc_return
1627 #endif
1628
1629 #endif /* arch_atomic64_inc_return_relaxed */
1630
1631 #ifndef arch_atomic64_fetch_inc_relaxed
1632 #ifdef arch_atomic64_fetch_inc
1633 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
1634 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
1635 #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
1636 #endif /* arch_atomic64_fetch_inc */
1637
1638 #ifndef arch_atomic64_fetch_inc
1639 static __always_inline s64
1640 arch_atomic64_fetch_inc(atomic64_t *v)
1641 {
1642         return arch_atomic64_fetch_add(1, v);
1643 }
1644 #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1645 #endif
1646
1647 #ifndef arch_atomic64_fetch_inc_acquire
1648 static __always_inline s64
1649 arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1650 {
1651         return arch_atomic64_fetch_add_acquire(1, v);
1652 }
1653 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1654 #endif
1655
1656 #ifndef arch_atomic64_fetch_inc_release
1657 static __always_inline s64
1658 arch_atomic64_fetch_inc_release(atomic64_t *v)
1659 {
1660         return arch_atomic64_fetch_add_release(1, v);
1661 }
1662 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1663 #endif
1664
1665 #ifndef arch_atomic64_fetch_inc_relaxed
1666 static __always_inline s64
1667 arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
1668 {
1669         return arch_atomic64_fetch_add_relaxed(1, v);
1670 }
1671 #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
1672 #endif
1673
1674 #else /* arch_atomic64_fetch_inc_relaxed */
1675
1676 #ifndef arch_atomic64_fetch_inc_acquire
1677 static __always_inline s64
1678 arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1679 {
1680         s64 ret = arch_atomic64_fetch_inc_relaxed(v);
1681         __atomic_acquire_fence();
1682         return ret;
1683 }
1684 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1685 #endif
1686
1687 #ifndef arch_atomic64_fetch_inc_release
1688 static __always_inline s64
1689 arch_atomic64_fetch_inc_release(atomic64_t *v)
1690 {
1691         __atomic_release_fence();
1692         return arch_atomic64_fetch_inc_relaxed(v);
1693 }
1694 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1695 #endif
1696
1697 #ifndef arch_atomic64_fetch_inc
1698 static __always_inline s64
1699 arch_atomic64_fetch_inc(atomic64_t *v)
1700 {
1701         s64 ret;
1702         __atomic_pre_full_fence();
1703         ret = arch_atomic64_fetch_inc_relaxed(v);
1704         __atomic_post_full_fence();
1705         return ret;
1706 }
1707 #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1708 #endif
1709
1710 #endif /* arch_atomic64_fetch_inc_relaxed */
1711
1712 #ifndef arch_atomic64_dec
1713 static __always_inline void
1714 arch_atomic64_dec(atomic64_t *v)
1715 {
1716         arch_atomic64_sub(1, v);
1717 }
1718 #define arch_atomic64_dec arch_atomic64_dec
1719 #endif
1720
1721 #ifndef arch_atomic64_dec_return_relaxed
1722 #ifdef arch_atomic64_dec_return
1723 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
1724 #define arch_atomic64_dec_return_release arch_atomic64_dec_return
1725 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
1726 #endif /* arch_atomic64_dec_return */
1727
1728 #ifndef arch_atomic64_dec_return
1729 static __always_inline s64
1730 arch_atomic64_dec_return(atomic64_t *v)
1731 {
1732         return arch_atomic64_sub_return(1, v);
1733 }
1734 #define arch_atomic64_dec_return arch_atomic64_dec_return
1735 #endif
1736
1737 #ifndef arch_atomic64_dec_return_acquire
1738 static __always_inline s64
1739 arch_atomic64_dec_return_acquire(atomic64_t *v)
1740 {
1741         return arch_atomic64_sub_return_acquire(1, v);
1742 }
1743 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1744 #endif
1745
1746 #ifndef arch_atomic64_dec_return_release
1747 static __always_inline s64
1748 arch_atomic64_dec_return_release(atomic64_t *v)
1749 {
1750         return arch_atomic64_sub_return_release(1, v);
1751 }
1752 #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1753 #endif
1754
1755 #ifndef arch_atomic64_dec_return_relaxed
1756 static __always_inline s64
1757 arch_atomic64_dec_return_relaxed(atomic64_t *v)
1758 {
1759         return arch_atomic64_sub_return_relaxed(1, v);
1760 }
1761 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
1762 #endif
1763
1764 #else /* arch_atomic64_dec_return_relaxed */
1765
1766 #ifndef arch_atomic64_dec_return_acquire
1767 static __always_inline s64
1768 arch_atomic64_dec_return_acquire(atomic64_t *v)
1769 {
1770         s64 ret = arch_atomic64_dec_return_relaxed(v);
1771         __atomic_acquire_fence();
1772         return ret;
1773 }
1774 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1775 #endif
1776
1777 #ifndef arch_atomic64_dec_return_release
1778 static __always_inline s64
1779 arch_atomic64_dec_return_release(atomic64_t *v)
1780 {
1781         __atomic_release_fence();
1782         return arch_atomic64_dec_return_relaxed(v);
1783 }
1784 #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1785 #endif
1786
1787 #ifndef arch_atomic64_dec_return
1788 static __always_inline s64
1789 arch_atomic64_dec_return(atomic64_t *v)
1790 {
1791         s64 ret;
1792         __atomic_pre_full_fence();
1793         ret = arch_atomic64_dec_return_relaxed(v);
1794         __atomic_post_full_fence();
1795         return ret;
1796 }
1797 #define arch_atomic64_dec_return arch_atomic64_dec_return
1798 #endif
1799
1800 #endif /* arch_atomic64_dec_return_relaxed */
1801
1802 #ifndef arch_atomic64_fetch_dec_relaxed
1803 #ifdef arch_atomic64_fetch_dec
1804 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
1805 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
1806 #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
1807 #endif /* arch_atomic64_fetch_dec */
1808
1809 #ifndef arch_atomic64_fetch_dec
1810 static __always_inline s64
1811 arch_atomic64_fetch_dec(atomic64_t *v)
1812 {
1813         return arch_atomic64_fetch_sub(1, v);
1814 }
1815 #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1816 #endif
1817
1818 #ifndef arch_atomic64_fetch_dec_acquire
1819 static __always_inline s64
1820 arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1821 {
1822         return arch_atomic64_fetch_sub_acquire(1, v);
1823 }
1824 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1825 #endif
1826
1827 #ifndef arch_atomic64_fetch_dec_release
1828 static __always_inline s64
1829 arch_atomic64_fetch_dec_release(atomic64_t *v)
1830 {
1831         return arch_atomic64_fetch_sub_release(1, v);
1832 }
1833 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1834 #endif
1835
1836 #ifndef arch_atomic64_fetch_dec_relaxed
1837 static __always_inline s64
1838 arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
1839 {
1840         return arch_atomic64_fetch_sub_relaxed(1, v);
1841 }
1842 #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
1843 #endif
1844
1845 #else /* arch_atomic64_fetch_dec_relaxed */
1846
1847 #ifndef arch_atomic64_fetch_dec_acquire
1848 static __always_inline s64
1849 arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1850 {
1851         s64 ret = arch_atomic64_fetch_dec_relaxed(v);
1852         __atomic_acquire_fence();
1853         return ret;
1854 }
1855 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1856 #endif
1857
1858 #ifndef arch_atomic64_fetch_dec_release
1859 static __always_inline s64
1860 arch_atomic64_fetch_dec_release(atomic64_t *v)
1861 {
1862         __atomic_release_fence();
1863         return arch_atomic64_fetch_dec_relaxed(v);
1864 }
1865 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1866 #endif
1867
1868 #ifndef arch_atomic64_fetch_dec
1869 static __always_inline s64
1870 arch_atomic64_fetch_dec(atomic64_t *v)
1871 {
1872         s64 ret;
1873         __atomic_pre_full_fence();
1874         ret = arch_atomic64_fetch_dec_relaxed(v);
1875         __atomic_post_full_fence();
1876         return ret;
1877 }
1878 #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1879 #endif
1880
1881 #endif /* arch_atomic64_fetch_dec_relaxed */
1882
1883 #ifndef arch_atomic64_fetch_and_relaxed
1884 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
1885 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
1886 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
1887 #else /* arch_atomic64_fetch_and_relaxed */
1888
1889 #ifndef arch_atomic64_fetch_and_acquire
1890 static __always_inline s64
1891 arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1892 {
1893         s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
1894         __atomic_acquire_fence();
1895         return ret;
1896 }
1897 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
1898 #endif
1899
1900 #ifndef arch_atomic64_fetch_and_release
1901 static __always_inline s64
1902 arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
1903 {
1904         __atomic_release_fence();
1905         return arch_atomic64_fetch_and_relaxed(i, v);
1906 }
1907 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
1908 #endif
1909
1910 #ifndef arch_atomic64_fetch_and
1911 static __always_inline s64
1912 arch_atomic64_fetch_and(s64 i, atomic64_t *v)
1913 {
1914         s64 ret;
1915         __atomic_pre_full_fence();
1916         ret = arch_atomic64_fetch_and_relaxed(i, v);
1917         __atomic_post_full_fence();
1918         return ret;
1919 }
1920 #define arch_atomic64_fetch_and arch_atomic64_fetch_and
1921 #endif
1922
1923 #endif /* arch_atomic64_fetch_and_relaxed */
1924
1925 #ifndef arch_atomic64_andnot
1926 static __always_inline void
1927 arch_atomic64_andnot(s64 i, atomic64_t *v)
1928 {
1929         arch_atomic64_and(~i, v);
1930 }
1931 #define arch_atomic64_andnot arch_atomic64_andnot
1932 #endif
1933
1934 #ifndef arch_atomic64_fetch_andnot_relaxed
1935 #ifdef arch_atomic64_fetch_andnot
1936 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
1937 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
1938 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
1939 #endif /* arch_atomic64_fetch_andnot */
1940
1941 #ifndef arch_atomic64_fetch_andnot
1942 static __always_inline s64
1943 arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1944 {
1945         return arch_atomic64_fetch_and(~i, v);
1946 }
1947 #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1948 #endif
1949
1950 #ifndef arch_atomic64_fetch_andnot_acquire
1951 static __always_inline s64
1952 arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1953 {
1954         return arch_atomic64_fetch_and_acquire(~i, v);
1955 }
1956 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1957 #endif
1958
1959 #ifndef arch_atomic64_fetch_andnot_release
1960 static __always_inline s64
1961 arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1962 {
1963         return arch_atomic64_fetch_and_release(~i, v);
1964 }
1965 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1966 #endif
1967
1968 #ifndef arch_atomic64_fetch_andnot_relaxed
1969 static __always_inline s64
1970 arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1971 {
1972         return arch_atomic64_fetch_and_relaxed(~i, v);
1973 }
1974 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
1975 #endif
1976
1977 #else /* arch_atomic64_fetch_andnot_relaxed */
1978
1979 #ifndef arch_atomic64_fetch_andnot_acquire
1980 static __always_inline s64
1981 arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1982 {
1983         s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1984         __atomic_acquire_fence();
1985         return ret;
1986 }
1987 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1988 #endif
1989
1990 #ifndef arch_atomic64_fetch_andnot_release
1991 static __always_inline s64
1992 arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1993 {
1994         __atomic_release_fence();
1995         return arch_atomic64_fetch_andnot_relaxed(i, v);
1996 }
1997 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1998 #endif
1999
2000 #ifndef arch_atomic64_fetch_andnot
2001 static __always_inline s64
2002 arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
2003 {
2004         s64 ret;
2005         __atomic_pre_full_fence();
2006         ret = arch_atomic64_fetch_andnot_relaxed(i, v);
2007         __atomic_post_full_fence();
2008         return ret;
2009 }
2010 #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
2011 #endif
2012
2013 #endif /* arch_atomic64_fetch_andnot_relaxed */
2014
2015 #ifndef arch_atomic64_fetch_or_relaxed
2016 #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
2017 #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
2018 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or
2019 #else /* arch_atomic64_fetch_or_relaxed */
2020
2021 #ifndef arch_atomic64_fetch_or_acquire
2022 static __always_inline s64
2023 arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
2024 {
2025         s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
2026         __atomic_acquire_fence();
2027         return ret;
2028 }
2029 #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
2030 #endif
2031
2032 #ifndef arch_atomic64_fetch_or_release
2033 static __always_inline s64
2034 arch_atomic64_fetch_or_release(s64 i, atomic64_t *v)
2035 {
2036         __atomic_release_fence();
2037         return arch_atomic64_fetch_or_relaxed(i, v);
2038 }
2039 #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
2040 #endif
2041
2042 #ifndef arch_atomic64_fetch_or
2043 static __always_inline s64
2044 arch_atomic64_fetch_or(s64 i, atomic64_t *v)
2045 {
2046         s64 ret;
2047         __atomic_pre_full_fence();
2048         ret = arch_atomic64_fetch_or_relaxed(i, v);
2049         __atomic_post_full_fence();
2050         return ret;
2051 }
2052 #define arch_atomic64_fetch_or arch_atomic64_fetch_or
2053 #endif
2054
2055 #endif /* arch_atomic64_fetch_or_relaxed */
2056
2057 #ifndef arch_atomic64_fetch_xor_relaxed
2058 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
2059 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
2060 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor
2061 #else /* arch_atomic64_fetch_xor_relaxed */
2062
2063 #ifndef arch_atomic64_fetch_xor_acquire
2064 static __always_inline s64
2065 arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
2066 {
2067         s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
2068         __atomic_acquire_fence();
2069         return ret;
2070 }
2071 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
2072 #endif
2073
2074 #ifndef arch_atomic64_fetch_xor_release
2075 static __always_inline s64
2076 arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
2077 {
2078         __atomic_release_fence();
2079         return arch_atomic64_fetch_xor_relaxed(i, v);
2080 }
2081 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
2082 #endif
2083
2084 #ifndef arch_atomic64_fetch_xor
2085 static __always_inline s64
2086 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
2087 {
2088         s64 ret;
2089         __atomic_pre_full_fence();
2090         ret = arch_atomic64_fetch_xor_relaxed(i, v);
2091         __atomic_post_full_fence();
2092         return ret;
2093 }
2094 #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
2095 #endif
2096
2097 #endif /* arch_atomic64_fetch_xor_relaxed */
2098
2099 #ifndef arch_atomic64_xchg_relaxed
2100 #define arch_atomic64_xchg_acquire arch_atomic64_xchg
2101 #define arch_atomic64_xchg_release arch_atomic64_xchg
2102 #define arch_atomic64_xchg_relaxed arch_atomic64_xchg
2103 #else /* arch_atomic64_xchg_relaxed */
2104
2105 #ifndef arch_atomic64_xchg_acquire
2106 static __always_inline s64
2107 arch_atomic64_xchg_acquire(atomic64_t *v, s64 i)
2108 {
2109         s64 ret = arch_atomic64_xchg_relaxed(v, i);
2110         __atomic_acquire_fence();
2111         return ret;
2112 }
2113 #define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire
2114 #endif
2115
2116 #ifndef arch_atomic64_xchg_release
2117 static __always_inline s64
2118 arch_atomic64_xchg_release(atomic64_t *v, s64 i)
2119 {
2120         __atomic_release_fence();
2121         return arch_atomic64_xchg_relaxed(v, i);
2122 }
2123 #define arch_atomic64_xchg_release arch_atomic64_xchg_release
2124 #endif
2125
2126 #ifndef arch_atomic64_xchg
2127 static __always_inline s64
2128 arch_atomic64_xchg(atomic64_t *v, s64 i)
2129 {
2130         s64 ret;
2131         __atomic_pre_full_fence();
2132         ret = arch_atomic64_xchg_relaxed(v, i);
2133         __atomic_post_full_fence();
2134         return ret;
2135 }
2136 #define arch_atomic64_xchg arch_atomic64_xchg
2137 #endif
2138
2139 #endif /* arch_atomic64_xchg_relaxed */
2140
2141 #ifndef arch_atomic64_cmpxchg_relaxed
2142 #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg
2143 #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg
2144 #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg
2145 #else /* arch_atomic64_cmpxchg_relaxed */
2146
2147 #ifndef arch_atomic64_cmpxchg_acquire
2148 static __always_inline s64
2149 arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
2150 {
2151         s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2152         __atomic_acquire_fence();
2153         return ret;
2154 }
2155 #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire
2156 #endif
2157
2158 #ifndef arch_atomic64_cmpxchg_release
2159 static __always_inline s64
2160 arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
2161 {
2162         __atomic_release_fence();
2163         return arch_atomic64_cmpxchg_relaxed(v, old, new);
2164 }
2165 #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release
2166 #endif
2167
2168 #ifndef arch_atomic64_cmpxchg
2169 static __always_inline s64
2170 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2171 {
2172         s64 ret;
2173         __atomic_pre_full_fence();
2174         ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2175         __atomic_post_full_fence();
2176         return ret;
2177 }
2178 #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
2179 #endif
2180
2181 #endif /* arch_atomic64_cmpxchg_relaxed */
2182
2183 #ifndef arch_atomic64_try_cmpxchg_relaxed
2184 #ifdef arch_atomic64_try_cmpxchg
2185 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg
2186 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg
2187 #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg
2188 #endif /* arch_atomic64_try_cmpxchg */
2189
2190 #ifndef arch_atomic64_try_cmpxchg
2191 static __always_inline bool
2192 arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2193 {
2194         s64 r, o = *old;
2195         r = arch_atomic64_cmpxchg(v, o, new);
2196         if (unlikely(r != o))
2197                 *old = r;
2198         return likely(r == o);
2199 }
2200 #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2201 #endif
2202
2203 #ifndef arch_atomic64_try_cmpxchg_acquire
2204 static __always_inline bool
2205 arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2206 {
2207         s64 r, o = *old;
2208         r = arch_atomic64_cmpxchg_acquire(v, o, new);
2209         if (unlikely(r != o))
2210                 *old = r;
2211         return likely(r == o);
2212 }
2213 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2214 #endif
2215
2216 #ifndef arch_atomic64_try_cmpxchg_release
2217 static __always_inline bool
2218 arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2219 {
2220         s64 r, o = *old;
2221         r = arch_atomic64_cmpxchg_release(v, o, new);
2222         if (unlikely(r != o))
2223                 *old = r;
2224         return likely(r == o);
2225 }
2226 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2227 #endif
2228
2229 #ifndef arch_atomic64_try_cmpxchg_relaxed
2230 static __always_inline bool
2231 arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2232 {
2233         s64 r, o = *old;
2234         r = arch_atomic64_cmpxchg_relaxed(v, o, new);
2235         if (unlikely(r != o))
2236                 *old = r;
2237         return likely(r == o);
2238 }
2239 #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed
2240 #endif
2241
2242 #else /* arch_atomic64_try_cmpxchg_relaxed */
2243
2244 #ifndef arch_atomic64_try_cmpxchg_acquire
2245 static __always_inline bool
2246 arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2247 {
2248         bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2249         __atomic_acquire_fence();
2250         return ret;
2251 }
2252 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2253 #endif
2254
2255 #ifndef arch_atomic64_try_cmpxchg_release
2256 static __always_inline bool
2257 arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2258 {
2259         __atomic_release_fence();
2260         return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2261 }
2262 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2263 #endif
2264
2265 #ifndef arch_atomic64_try_cmpxchg
2266 static __always_inline bool
2267 arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2268 {
2269         bool ret;
2270         __atomic_pre_full_fence();
2271         ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2272         __atomic_post_full_fence();
2273         return ret;
2274 }
2275 #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2276 #endif
2277
2278 #endif /* arch_atomic64_try_cmpxchg_relaxed */
2279
2280 #ifndef arch_atomic64_sub_and_test
2281 /**
2282  * arch_atomic64_sub_and_test - subtract value from variable and test result
2283  * @i: integer value to subtract
2284  * @v: pointer of type atomic64_t
2285  *
2286  * Atomically subtracts @i from @v and returns
2287  * true if the result is zero, or false for all
2288  * other cases.
2289  */
2290 static __always_inline bool
2291 arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
2292 {
2293         return arch_atomic64_sub_return(i, v) == 0;
2294 }
2295 #define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
2296 #endif
2297
2298 #ifndef arch_atomic64_dec_and_test
2299 /**
2300  * arch_atomic64_dec_and_test - decrement and test
2301  * @v: pointer of type atomic64_t
2302  *
2303  * Atomically decrements @v by 1 and
2304  * returns true if the result is 0, or false for all other
2305  * cases.
2306  */
2307 static __always_inline bool
2308 arch_atomic64_dec_and_test(atomic64_t *v)
2309 {
2310         return arch_atomic64_dec_return(v) == 0;
2311 }
2312 #define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
2313 #endif
2314
2315 #ifndef arch_atomic64_inc_and_test
2316 /**
2317  * arch_atomic64_inc_and_test - increment and test
2318  * @v: pointer of type atomic64_t
2319  *
2320  * Atomically increments @v by 1
2321  * and returns true if the result is zero, or false for all
2322  * other cases.
2323  */
2324 static __always_inline bool
2325 arch_atomic64_inc_and_test(atomic64_t *v)
2326 {
2327         return arch_atomic64_inc_return(v) == 0;
2328 }
2329 #define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
2330 #endif
2331
2332 #ifndef arch_atomic64_add_negative
2333 /**
2334  * arch_atomic64_add_negative - add and test if negative
2335  * @i: integer value to add
2336  * @v: pointer of type atomic64_t
2337  *
2338  * Atomically adds @i to @v and returns true
2339  * if the result is negative, or false when
2340  * result is greater than or equal to zero.
2341  */
2342 static __always_inline bool
2343 arch_atomic64_add_negative(s64 i, atomic64_t *v)
2344 {
2345         return arch_atomic64_add_return(i, v) < 0;
2346 }
2347 #define arch_atomic64_add_negative arch_atomic64_add_negative
2348 #endif
2349
2350 #ifndef arch_atomic64_fetch_add_unless
2351 /**
2352  * arch_atomic64_fetch_add_unless - add unless the number is already a given value
2353  * @v: pointer of type atomic64_t
2354  * @a: the amount to add to v...
2355  * @u: ...unless v is equal to u.
2356  *
2357  * Atomically adds @a to @v, so long as @v was not already @u.
2358  * Returns original value of @v
2359  */
2360 static __always_inline s64
2361 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2362 {
2363         s64 c = arch_atomic64_read(v);
2364
2365         do {
2366                 if (unlikely(c == u))
2367                         break;
2368         } while (!arch_atomic64_try_cmpxchg(v, &c, c + a));
2369
2370         return c;
2371 }
2372 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
2373 #endif
2374
2375 #ifndef arch_atomic64_add_unless
2376 /**
2377  * arch_atomic64_add_unless - add unless the number is already a given value
2378  * @v: pointer of type atomic64_t
2379  * @a: the amount to add to v...
2380  * @u: ...unless v is equal to u.
2381  *
2382  * Atomically adds @a to @v, if @v was not already @u.
2383  * Returns true if the addition was done.
2384  */
2385 static __always_inline bool
2386 arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2387 {
2388         return arch_atomic64_fetch_add_unless(v, a, u) != u;
2389 }
2390 #define arch_atomic64_add_unless arch_atomic64_add_unless
2391 #endif
2392
2393 #ifndef arch_atomic64_inc_not_zero
2394 /**
2395  * arch_atomic64_inc_not_zero - increment unless the number is zero
2396  * @v: pointer of type atomic64_t
2397  *
2398  * Atomically increments @v by 1, if @v is non-zero.
2399  * Returns true if the increment was done.
2400  */
2401 static __always_inline bool
2402 arch_atomic64_inc_not_zero(atomic64_t *v)
2403 {
2404         return arch_atomic64_add_unless(v, 1, 0);
2405 }
2406 #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
2407 #endif
2408
2409 #ifndef arch_atomic64_inc_unless_negative
2410 static __always_inline bool
2411 arch_atomic64_inc_unless_negative(atomic64_t *v)
2412 {
2413         s64 c = arch_atomic64_read(v);
2414
2415         do {
2416                 if (unlikely(c < 0))
2417                         return false;
2418         } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1));
2419
2420         return true;
2421 }
2422 #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
2423 #endif
2424
2425 #ifndef arch_atomic64_dec_unless_positive
2426 static __always_inline bool
2427 arch_atomic64_dec_unless_positive(atomic64_t *v)
2428 {
2429         s64 c = arch_atomic64_read(v);
2430
2431         do {
2432                 if (unlikely(c > 0))
2433                         return false;
2434         } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1));
2435
2436         return true;
2437 }
2438 #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
2439 #endif
2440
2441 #ifndef arch_atomic64_dec_if_positive
2442 static __always_inline s64
2443 arch_atomic64_dec_if_positive(atomic64_t *v)
2444 {
2445         s64 dec, c = arch_atomic64_read(v);
2446
2447         do {
2448                 dec = c - 1;
2449                 if (unlikely(dec < 0))
2450                         break;
2451         } while (!arch_atomic64_try_cmpxchg(v, &c, dec));
2452
2453         return dec;
2454 }
2455 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
2456 #endif
2457
2458 #endif /* _LINUX_ATOMIC_FALLBACK_H */
2459 // b5e87bdd5ede61470c29f7a7e4de781af3770f09