arch/arc/include/asm/spinlock.h

   1 /*
   2  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #ifndef __ASM_SPINLOCK_H
  10 #define __ASM_SPINLOCK_H
  11
  12 #include <asm/spinlock_types.h>
  13 #include <asm/processor.h>
  14 #include <asm/barrier.h>
  15
  16 #define arch_spin_is_locked(x)  ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__)
  17
  18 #ifdef CONFIG_ARC_HAS_LLSC
  19
  20 static inline void arch_spin_lock(arch_spinlock_t *lock)
  21 {
  22         unsigned int val;
  23
  24         __asm__ __volatile__(
  25         "1:     llock   %[val], [%[slock]]      \n"
  26         "       breq    %[val], %[LOCKED], 1b   \n"     /* spin while LOCKED */
  27         "       scond   %[LOCKED], [%[slock]]   \n"     /* acquire */
  28         "       bnz     1b                      \n"
  29         "                                       \n"
  30         : [val]         "=&r"   (val)
  31         : [slock]       "r"     (&(lock->slock)),
  32           [LOCKED]      "r"     (__ARCH_SPIN_LOCK_LOCKED__)
  33         : "memory", "cc");
  34
  35         /*
  36          * ACQUIRE barrier to ensure load/store after taking the lock
  37          * don't "bleed-up" out of the critical section (leak-in is allowed)
  38          * http://www.spinics.net/lists/kernel/msg2010409.html
  39          *
  40          * ARCv2 only has load-load, store-store and all-all barrier
  41          * thus need the full all-all barrier
  42          */
  43         smp_mb();
  44 }
  45
  46 /* 1 - lock taken successfully */
  47 static inline int arch_spin_trylock(arch_spinlock_t *lock)
  48 {
  49         unsigned int val, got_it = 0;
  50
  51         __asm__ __volatile__(
  52         "1:     llock   %[val], [%[slock]]      \n"
  53         "       breq    %[val], %[LOCKED], 4f   \n"     /* already LOCKED, just bail */
  54         "       scond   %[LOCKED], [%[slock]]   \n"     /* acquire */
  55         "       bnz     1b                      \n"
  56         "       mov     %[got_it], 1            \n"
  57         "4:                                     \n"
  58         "                                       \n"
  59         : [val]         "=&r"   (val),
  60           [got_it]      "+&r"   (got_it)
  61         : [slock]       "r"     (&(lock->slock)),
  62           [LOCKED]      "r"     (__ARCH_SPIN_LOCK_LOCKED__)
  63         : "memory", "cc");
  64
  65         smp_mb();
  66
  67         return got_it;
  68 }
  69
  70 static inline void arch_spin_unlock(arch_spinlock_t *lock)
  71 {
  72         smp_mb();
  73
  74         WRITE_ONCE(lock->slock, __ARCH_SPIN_LOCK_UNLOCKED__);
  75 }
  76
  77 /*
  78  * Read-write spinlocks, allowing multiple readers but only one writer.
  79  * Unfair locking as Writers could be starved indefinitely by Reader(s)
  80  */
  81
  82 static inline void arch_read_lock(arch_rwlock_t *rw)
  83 {
  84         unsigned int val;
  85
  86         /*
  87          * zero means writer holds the lock exclusively, deny Reader.
  88          * Otherwise grant lock to first/subseq reader
  89          *
  90          *      if (rw->counter > 0) {
  91          *              rw->counter--;
  92          *              ret = 1;
  93          *      }
  94          */
  95
  96         __asm__ __volatile__(
  97         "1:     llock   %[val], [%[rwlock]]     \n"
  98         "       brls    %[val], %[WR_LOCKED], 1b\n"     /* <= 0: spin while write locked */
  99         "       sub     %[val], %[val], 1       \n"     /* reader lock */
 100         "       scond   %[val], [%[rwlock]]     \n"
 101         "       bnz     1b                      \n"
 102         "                                       \n"
 103         : [val]         "=&r"   (val)
 104         : [rwlock]      "r"     (&(rw->counter)),
 105           [WR_LOCKED]   "ir"    (0)
 106         : "memory", "cc");
 107
 108         smp_mb();
 109 }
 110
 111 /* 1 - lock taken successfully */
 112 static inline int arch_read_trylock(arch_rwlock_t *rw)
 113 {
 114         unsigned int val, got_it = 0;
 115
 116         __asm__ __volatile__(
 117         "1:     llock   %[val], [%[rwlock]]     \n"
 118         "       brls    %[val], %[WR_LOCKED], 4f\n"     /* <= 0: already write locked, bail */
 119         "       sub     %[val], %[val], 1       \n"     /* counter-- */
 120         "       scond   %[val], [%[rwlock]]     \n"
 121         "       bnz     1b                      \n"     /* retry if collided with someone */
 122         "       mov     %[got_it], 1            \n"
 123         "                                       \n"
 124         "4: ; --- done ---                      \n"
 125
 126         : [val]         "=&r"   (val),
 127           [got_it]      "+&r"   (got_it)
 128         : [rwlock]      "r"     (&(rw->counter)),
 129           [WR_LOCKED]   "ir"    (0)
 130         : "memory", "cc");
 131
 132         smp_mb();
 133
 134         return got_it;
 135 }
 136
 137 static inline void arch_write_lock(arch_rwlock_t *rw)
 138 {
 139         unsigned int val;
 140
 141         /*
 142          * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
 143          * deny writer. Otherwise if unlocked grant to writer
 144          * Hence the claim that Linux rwlocks are unfair to writers.
 145          * (can be starved for an indefinite time by readers).
 146          *
 147          *      if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
 148          *              rw->counter = 0;
 149          *              ret = 1;
 150          *      }
 151          */
 152
 153         __asm__ __volatile__(
 154         "1:     llock   %[val], [%[rwlock]]     \n"
 155         "       brne    %[val], %[UNLOCKED], 1b \n"     /* while !UNLOCKED spin */
 156         "       mov     %[val], %[WR_LOCKED]    \n"
 157         "       scond   %[val], [%[rwlock]]     \n"
 158         "       bnz     1b                      \n"
 159         "                                       \n"
 160         : [val]         "=&r"   (val)
 161         : [rwlock]      "r"     (&(rw->counter)),
 162           [UNLOCKED]    "ir"    (__ARCH_RW_LOCK_UNLOCKED__),
 163           [WR_LOCKED]   "ir"    (0)
 164         : "memory", "cc");
 165
 166         smp_mb();
 167 }
 168
 169 /* 1 - lock taken successfully */
 170 static inline int arch_write_trylock(arch_rwlock_t *rw)
 171 {
 172         unsigned int val, got_it = 0;
 173
 174         __asm__ __volatile__(
 175         "1:     llock   %[val], [%[rwlock]]     \n"
 176         "       brne    %[val], %[UNLOCKED], 4f \n"     /* !UNLOCKED, bail */
 177         "       mov     %[val], %[WR_LOCKED]    \n"
 178         "       scond   %[val], [%[rwlock]]     \n"
 179         "       bnz     1b                      \n"     /* retry if collided with someone */
 180         "       mov     %[got_it], 1            \n"
 181         "                                       \n"
 182         "4: ; --- done ---                      \n"
 183
 184         : [val]         "=&r"   (val),
 185           [got_it]      "+&r"   (got_it)
 186         : [rwlock]      "r"     (&(rw->counter)),
 187           [UNLOCKED]    "ir"    (__ARCH_RW_LOCK_UNLOCKED__),
 188           [WR_LOCKED]   "ir"    (0)
 189         : "memory", "cc");
 190
 191         smp_mb();
 192
 193         return got_it;
 194 }
 195
 196 static inline void arch_read_unlock(arch_rwlock_t *rw)
 197 {
 198         unsigned int val;
 199
 200         smp_mb();
 201
 202         /*
 203          * rw->counter++;
 204          */
 205         __asm__ __volatile__(
 206         "1:     llock   %[val], [%[rwlock]]     \n"
 207         "       add     %[val], %[val], 1       \n"
 208         "       scond   %[val], [%[rwlock]]     \n"
 209         "       bnz     1b                      \n"
 210         "                                       \n"
 211         : [val]         "=&r"   (val)
 212         : [rwlock]      "r"     (&(rw->counter))
 213         : "memory", "cc");
 214 }
 215
 216 static inline void arch_write_unlock(arch_rwlock_t *rw)
 217 {
 218         smp_mb();
 219
 220         WRITE_ONCE(rw->counter, __ARCH_RW_LOCK_UNLOCKED__);
 221 }
 222
 223 #else   /* !CONFIG_ARC_HAS_LLSC */
 224
 225 static inline void arch_spin_lock(arch_spinlock_t *lock)
 226 {
 227         unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
 228
 229         /*
 230          * Per lkmm, smp_mb() is only required after _lock (and before_unlock)
 231          * for ACQ and REL semantics respectively. However EX based spinlocks
 232          * need the extra smp_mb to workaround a hardware quirk.
 233          */
 234         smp_mb();
 235
 236         __asm__ __volatile__(
 237         "1:     ex  %0, [%1]            \n"
 238 #ifdef CONFIG_EZNPS_MTM_EXT
 239         "       .word %3                \n"
 240 #endif
 241         "       breq  %0, %2, 1b        \n"
 242         : "+&r" (val)
 243         : "r"(&(lock->slock)), "ir"(__ARCH_SPIN_LOCK_LOCKED__)
 244 #ifdef CONFIG_EZNPS_MTM_EXT
 245         , "i"(CTOP_INST_SCHD_RW)
 246 #endif
 247         : "memory");
 248
 249         smp_mb();
 250 }
 251
 252 /* 1 - lock taken successfully */
 253 static inline int arch_spin_trylock(arch_spinlock_t *lock)
 254 {
 255         unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
 256
 257         smp_mb();
 258
 259         __asm__ __volatile__(
 260         "1:     ex  %0, [%1]            \n"
 261         : "+r" (val)
 262         : "r"(&(lock->slock))
 263         : "memory");
 264
 265         smp_mb();
 266
 267         return (val == __ARCH_SPIN_LOCK_UNLOCKED__);
 268 }
 269
 270 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 271 {
 272         unsigned int val = __ARCH_SPIN_LOCK_UNLOCKED__;
 273
 274         /*
 275          * RELEASE barrier: given the instructions avail on ARCv2, full barrier
 276          * is the only option
 277          */
 278         smp_mb();
 279
 280         /*
 281          * EX is not really required here, a simple STore of 0 suffices.
 282          * However this causes tasklist livelocks in SystemC based SMP virtual
 283          * platforms where the systemc core scheduler uses EX as a cue for
 284          * moving to next core. Do a git log of this file for details
 285          */
 286         __asm__ __volatile__(
 287         "       ex  %0, [%1]            \n"
 288         : "+r" (val)
 289         : "r"(&(lock->slock))
 290         : "memory");
 291
 292         /*
 293          * see pairing version/comment in arch_spin_lock above
 294          */
 295         smp_mb();
 296 }
 297
 298 /*
 299  * Read-write spinlocks, allowing multiple readers but only one writer.
 300  * Unfair locking as Writers could be starved indefinitely by Reader(s)
 301  *
 302  * The spinlock itself is contained in @counter and access to it is
 303  * serialized with @lock_mutex.
 304  */
 305
 306 /* 1 - lock taken successfully */
 307 static inline int arch_read_trylock(arch_rwlock_t *rw)
 308 {
 309         int ret = 0;
 310         unsigned long flags;
 311
 312         local_irq_save(flags);
 313         arch_spin_lock(&(rw->lock_mutex));
 314
 315         /*
 316          * zero means writer holds the lock exclusively, deny Reader.
 317          * Otherwise grant lock to first/subseq reader
 318          */
 319         if (rw->counter > 0) {
 320                 rw->counter--;
 321                 ret = 1;
 322         }
 323
 324         arch_spin_unlock(&(rw->lock_mutex));
 325         local_irq_restore(flags);
 326
 327         return ret;
 328 }
 329
 330 /* 1 - lock taken successfully */
 331 static inline int arch_write_trylock(arch_rwlock_t *rw)
 332 {
 333         int ret = 0;
 334         unsigned long flags;
 335
 336         local_irq_save(flags);
 337         arch_spin_lock(&(rw->lock_mutex));
 338
 339         /*
 340          * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
 341          * deny writer. Otherwise if unlocked grant to writer
 342          * Hence the claim that Linux rwlocks are unfair to writers.
 343          * (can be starved for an indefinite time by readers).
 344          */
 345         if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
 346                 rw->counter = 0;
 347                 ret = 1;
 348         }
 349         arch_spin_unlock(&(rw->lock_mutex));
 350         local_irq_restore(flags);
 351
 352         return ret;
 353 }
 354
 355 static inline void arch_read_lock(arch_rwlock_t *rw)
 356 {
 357         while (!arch_read_trylock(rw))
 358                 cpu_relax();
 359 }
 360
 361 static inline void arch_write_lock(arch_rwlock_t *rw)
 362 {
 363         while (!arch_write_trylock(rw))
 364                 cpu_relax();
 365 }
 366
 367 static inline void arch_read_unlock(arch_rwlock_t *rw)
 368 {
 369         unsigned long flags;
 370
 371         local_irq_save(flags);
 372         arch_spin_lock(&(rw->lock_mutex));
 373         rw->counter++;
 374         arch_spin_unlock(&(rw->lock_mutex));
 375         local_irq_restore(flags);
 376 }
 377
 378 static inline void arch_write_unlock(arch_rwlock_t *rw)
 379 {
 380         unsigned long flags;
 381
 382         local_irq_save(flags);
 383         arch_spin_lock(&(rw->lock_mutex));
 384         rw->counter = __ARCH_RW_LOCK_UNLOCKED__;
 385         arch_spin_unlock(&(rw->lock_mutex));
 386         local_irq_restore(flags);
 387 }
 388
 389 #endif
 390
 391 #endif /* __ASM_SPINLOCK_H */