Commit | Line | Data |
---|---|---|
08e875c1 CM |
1 | /* |
2 | * Copyright (C) 2012 ARM Ltd. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | * GNU General Public License for more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public License | |
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
15 | */ | |
16 | #ifndef __ASM_SPINLOCK_H | |
17 | #define __ASM_SPINLOCK_H | |
18 | ||
19 | #include <asm/spinlock_types.h> | |
20 | #include <asm/processor.h> | |
21 | ||
22 | /* | |
23 | * Spinlock implementation. | |
24 | * | |
08e875c1 CM |
25 | * The memory barriers are implicit with the load-acquire and store-release |
26 | * instructions. | |
08e875c1 CM |
27 | */ |
28 | ||
08e875c1 CM |
29 | #define arch_spin_unlock_wait(lock) \ |
30 | do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) | |
31 | ||
32 | #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) | |
33 | ||
34 | static inline void arch_spin_lock(arch_spinlock_t *lock) | |
35 | { | |
36 | unsigned int tmp; | |
52ea2a56 | 37 | arch_spinlock_t lockval, newval; |
08e875c1 CM |
38 | |
39 | asm volatile( | |
52ea2a56 WD |
40 | /* Atomically increment the next ticket. */ |
41 | " prfm pstl1strm, %3\n" | |
42 | "1: ldaxr %w0, %3\n" | |
43 | " add %w1, %w0, %w5\n" | |
44 | " stxr %w2, %w1, %3\n" | |
45 | " cbnz %w2, 1b\n" | |
46 | /* Did we get the lock? */ | |
47 | " eor %w1, %w0, %w0, ror #16\n" | |
48 | " cbz %w1, 3f\n" | |
49 | /* | |
50 | * No: spin on the owner. Send a local event to avoid missing an | |
51 | * unlock before the exclusive load. | |
52 | */ | |
53 | " sevl\n" | |
54 | "2: wfe\n" | |
55 | " ldaxrh %w2, %4\n" | |
56 | " eor %w1, %w2, %w0, lsr #16\n" | |
57 | " cbnz %w1, 2b\n" | |
58 | /* We got the lock. Critical section starts here. */ | |
59 | "3:" | |
60 | : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) | |
61 | : "Q" (lock->owner), "I" (1 << TICKET_SHIFT) | |
62 | : "memory"); | |
08e875c1 CM |
63 | } |
64 | ||
65 | static inline int arch_spin_trylock(arch_spinlock_t *lock) | |
66 | { | |
67 | unsigned int tmp; | |
52ea2a56 | 68 | arch_spinlock_t lockval; |
08e875c1 CM |
69 | |
70 | asm volatile( | |
52ea2a56 WD |
71 | " prfm pstl1strm, %2\n" |
72 | "1: ldaxr %w0, %2\n" | |
73 | " eor %w1, %w0, %w0, ror #16\n" | |
74 | " cbnz %w1, 2f\n" | |
75 | " add %w0, %w0, %3\n" | |
76 | " stxr %w1, %w0, %2\n" | |
77 | " cbnz %w1, 1b\n" | |
78 | "2:" | |
79 | : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) | |
80 | : "I" (1 << TICKET_SHIFT) | |
81 | : "memory"); | |
08e875c1 CM |
82 | |
83 | return !tmp; | |
84 | } | |
85 | ||
86 | static inline void arch_spin_unlock(arch_spinlock_t *lock) | |
87 | { | |
88 | asm volatile( | |
52ea2a56 WD |
89 | " stlrh %w1, %0\n" |
90 | : "=Q" (lock->owner) | |
91 | : "r" (lock->owner + 1) | |
92 | : "memory"); | |
93 | } | |
94 | ||
5686b06c WD |
95 | static inline int arch_spin_value_unlocked(arch_spinlock_t lock) |
96 | { | |
97 | return lock.owner == lock.next; | |
98 | } | |
99 | ||
52ea2a56 WD |
100 | static inline int arch_spin_is_locked(arch_spinlock_t *lock) |
101 | { | |
5686b06c | 102 | return !arch_spin_value_unlocked(ACCESS_ONCE(*lock)); |
52ea2a56 WD |
103 | } |
104 | ||
105 | static inline int arch_spin_is_contended(arch_spinlock_t *lock) | |
106 | { | |
107 | arch_spinlock_t lockval = ACCESS_ONCE(*lock); | |
108 | return (lockval.next - lockval.owner) > 1; | |
08e875c1 | 109 | } |
52ea2a56 | 110 | #define arch_spin_is_contended arch_spin_is_contended |
08e875c1 CM |
111 | |
112 | /* | |
113 | * Write lock implementation. | |
114 | * | |
115 | * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is | |
116 | * exclusively held. | |
117 | * | |
118 | * The memory barriers are implicit with the load-acquire and store-release | |
119 | * instructions. | |
120 | */ | |
121 | ||
122 | static inline void arch_write_lock(arch_rwlock_t *rw) | |
123 | { | |
124 | unsigned int tmp; | |
125 | ||
126 | asm volatile( | |
127 | " sevl\n" | |
128 | "1: wfe\n" | |
3a0310eb | 129 | "2: ldaxr %w0, %1\n" |
08e875c1 | 130 | " cbnz %w0, 1b\n" |
3a0310eb | 131 | " stxr %w0, %w2, %1\n" |
08e875c1 | 132 | " cbnz %w0, 2b\n" |
3a0310eb WD |
133 | : "=&r" (tmp), "+Q" (rw->lock) |
134 | : "r" (0x80000000) | |
135 | : "cc", "memory"); | |
08e875c1 CM |
136 | } |
137 | ||
138 | static inline int arch_write_trylock(arch_rwlock_t *rw) | |
139 | { | |
140 | unsigned int tmp; | |
141 | ||
142 | asm volatile( | |
3a0310eb | 143 | " ldaxr %w0, %1\n" |
08e875c1 | 144 | " cbnz %w0, 1f\n" |
3a0310eb | 145 | " stxr %w0, %w2, %1\n" |
08e875c1 | 146 | "1:\n" |
3a0310eb WD |
147 | : "=&r" (tmp), "+Q" (rw->lock) |
148 | : "r" (0x80000000) | |
149 | : "cc", "memory"); | |
08e875c1 CM |
150 | |
151 | return !tmp; | |
152 | } | |
153 | ||
154 | static inline void arch_write_unlock(arch_rwlock_t *rw) | |
155 | { | |
156 | asm volatile( | |
3a0310eb WD |
157 | " stlr %w1, %0\n" |
158 | : "=Q" (rw->lock) : "r" (0) : "memory"); | |
08e875c1 CM |
159 | } |
160 | ||
161 | /* write_can_lock - would write_trylock() succeed? */ | |
162 | #define arch_write_can_lock(x) ((x)->lock == 0) | |
163 | ||
164 | /* | |
165 | * Read lock implementation. | |
166 | * | |
167 | * It exclusively loads the lock value, increments it and stores the new value | |
168 | * back if positive and the CPU still exclusively owns the location. If the | |
169 | * value is negative, the lock is already held. | |
170 | * | |
171 | * During unlocking there may be multiple active read locks but no write lock. | |
172 | * | |
173 | * The memory barriers are implicit with the load-acquire and store-release | |
174 | * instructions. | |
175 | */ | |
176 | static inline void arch_read_lock(arch_rwlock_t *rw) | |
177 | { | |
178 | unsigned int tmp, tmp2; | |
179 | ||
180 | asm volatile( | |
181 | " sevl\n" | |
182 | "1: wfe\n" | |
3a0310eb | 183 | "2: ldaxr %w0, %2\n" |
08e875c1 CM |
184 | " add %w0, %w0, #1\n" |
185 | " tbnz %w0, #31, 1b\n" | |
3a0310eb | 186 | " stxr %w1, %w0, %2\n" |
08e875c1 | 187 | " cbnz %w1, 2b\n" |
3a0310eb WD |
188 | : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) |
189 | : | |
190 | : "cc", "memory"); | |
08e875c1 CM |
191 | } |
192 | ||
193 | static inline void arch_read_unlock(arch_rwlock_t *rw) | |
194 | { | |
195 | unsigned int tmp, tmp2; | |
196 | ||
197 | asm volatile( | |
3a0310eb | 198 | "1: ldxr %w0, %2\n" |
08e875c1 | 199 | " sub %w0, %w0, #1\n" |
3a0310eb | 200 | " stlxr %w1, %w0, %2\n" |
08e875c1 | 201 | " cbnz %w1, 1b\n" |
3a0310eb WD |
202 | : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) |
203 | : | |
204 | : "cc", "memory"); | |
08e875c1 CM |
205 | } |
206 | ||
207 | static inline int arch_read_trylock(arch_rwlock_t *rw) | |
208 | { | |
209 | unsigned int tmp, tmp2 = 1; | |
210 | ||
211 | asm volatile( | |
3a0310eb | 212 | " ldaxr %w0, %2\n" |
08e875c1 CM |
213 | " add %w0, %w0, #1\n" |
214 | " tbnz %w0, #31, 1f\n" | |
3a0310eb | 215 | " stxr %w1, %w0, %2\n" |
08e875c1 | 216 | "1:\n" |
3a0310eb WD |
217 | : "=&r" (tmp), "+r" (tmp2), "+Q" (rw->lock) |
218 | : | |
219 | : "cc", "memory"); | |
08e875c1 CM |
220 | |
221 | return !tmp2; | |
222 | } | |
223 | ||
224 | /* read_can_lock - would read_trylock() succeed? */ | |
225 | #define arch_read_can_lock(x) ((x)->lock < 0x80000000) | |
226 | ||
227 | #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) | |
228 | #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) | |
229 | ||
230 | #define arch_spin_relax(lock) cpu_relax() | |
231 | #define arch_read_relax(lock) cpu_relax() | |
232 | #define arch_write_relax(lock) cpu_relax() | |
233 | ||
234 | #endif /* __ASM_SPINLOCK_H */ |