Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
102fa15c | 6 | * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org) |
1da177e4 LT |
7 | * Copyright (c) 1999, 2000 Silicon Graphics, Inc. |
8 | */ | |
9 | #ifndef _ASM_BITOPS_H | |
10 | #define _ASM_BITOPS_H | |
11 | ||
0624517d JS |
12 | #ifndef _LINUX_BITOPS_H |
13 | #error only <linux/bitops.h> can be included directly | |
14 | #endif | |
15 | ||
1da177e4 | 16 | #include <linux/compiler.h> |
4ffd8b38 | 17 | #include <linux/irqflags.h> |
1da177e4 | 18 | #include <linux/types.h> |
0004a9df | 19 | #include <asm/barrier.h> |
ec917c2c | 20 | #include <asm/bug.h> |
1da177e4 LT |
21 | #include <asm/byteorder.h> /* sigh ... */ |
22 | #include <asm/cpu-features.h> | |
4ffd8b38 RB |
23 | #include <asm/sgidefs.h> |
24 | #include <asm/war.h> | |
1da177e4 | 25 | |
49a89efb | 26 | #if _MIPS_SZLONG == 32 |
1da177e4 LT |
27 | #define SZLONG_LOG 5 |
28 | #define SZLONG_MASK 31UL | |
aac8aa77 MR |
29 | #define __LL "ll " |
30 | #define __SC "sc " | |
102fa15c RB |
31 | #define __INS "ins " |
32 | #define __EXT "ext " | |
49a89efb | 33 | #elif _MIPS_SZLONG == 64 |
1da177e4 LT |
34 | #define SZLONG_LOG 6 |
35 | #define SZLONG_MASK 63UL | |
aac8aa77 MR |
36 | #define __LL "lld " |
37 | #define __SC "scd " | |
102fa15c RB |
38 | #define __INS "dins " |
39 | #define __EXT "dext " | |
1da177e4 LT |
40 | #endif |
41 | ||
1da177e4 LT |
42 | /* |
43 | * clear_bit() doesn't provide any barrier for the compiler. | |
44 | */ | |
17099b11 RB |
45 | #define smp_mb__before_clear_bit() smp_llsc_mb() |
46 | #define smp_mb__after_clear_bit() smp_llsc_mb() | |
1da177e4 | 47 | |
1da177e4 LT |
48 | /* |
49 | * set_bit - Atomically set a bit in memory | |
50 | * @nr: the bit to set | |
51 | * @addr: the address to start counting from | |
52 | * | |
53 | * This function is atomic and may not be reordered. See __set_bit() | |
54 | * if you do not require the atomic guarantees. | |
55 | * Note that @nr may be almost arbitrarily large; this function is not | |
56 | * restricted to acting on a single-word quantity. | |
57 | */ | |
58 | static inline void set_bit(unsigned long nr, volatile unsigned long *addr) | |
59 | { | |
60 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
b961153b | 61 | unsigned short bit = nr & SZLONG_MASK; |
1da177e4 LT |
62 | unsigned long temp; |
63 | ||
64 | if (cpu_has_llsc && R10000_LLSC_WAR) { | |
65 | __asm__ __volatile__( | |
c4559f67 | 66 | " .set mips3 \n" |
1da177e4 LT |
67 | "1: " __LL "%0, %1 # set_bit \n" |
68 | " or %0, %2 \n" | |
aac8aa77 | 69 | " " __SC "%0, %1 \n" |
1da177e4 | 70 | " beqzl %0, 1b \n" |
aac8aa77 | 71 | " .set mips0 \n" |
1da177e4 | 72 | : "=&r" (temp), "=m" (*m) |
b961153b | 73 | : "ir" (1UL << bit), "m" (*m)); |
102fa15c | 74 | #ifdef CONFIG_CPU_MIPSR2 |
b961153b | 75 | } else if (__builtin_constant_p(bit)) { |
102fa15c RB |
76 | __asm__ __volatile__( |
77 | "1: " __LL "%0, %1 # set_bit \n" | |
78 | " " __INS "%0, %4, %2, 1 \n" | |
79 | " " __SC "%0, %1 \n" | |
80 | " beqz %0, 2f \n" | |
81 | " .subsection 2 \n" | |
82 | "2: b 1b \n" | |
83 | " .previous \n" | |
84 | : "=&r" (temp), "=m" (*m) | |
d6d8a463 | 85 | : "ir" (bit), "m" (*m), "r" (~0)); |
102fa15c | 86 | #endif /* CONFIG_CPU_MIPSR2 */ |
1da177e4 LT |
87 | } else if (cpu_has_llsc) { |
88 | __asm__ __volatile__( | |
c4559f67 | 89 | " .set mips3 \n" |
1da177e4 LT |
90 | "1: " __LL "%0, %1 # set_bit \n" |
91 | " or %0, %2 \n" | |
aac8aa77 | 92 | " " __SC "%0, %1 \n" |
f65e4fa8 RB |
93 | " beqz %0, 2f \n" |
94 | " .subsection 2 \n" | |
95 | "2: b 1b \n" | |
96 | " .previous \n" | |
aac8aa77 | 97 | " .set mips0 \n" |
1da177e4 | 98 | : "=&r" (temp), "=m" (*m) |
b961153b | 99 | : "ir" (1UL << bit), "m" (*m)); |
1da177e4 LT |
100 | } else { |
101 | volatile unsigned long *a = addr; | |
102 | unsigned long mask; | |
4ffd8b38 | 103 | unsigned long flags; |
1da177e4 LT |
104 | |
105 | a += nr >> SZLONG_LOG; | |
b961153b | 106 | mask = 1UL << bit; |
49edd098 | 107 | raw_local_irq_save(flags); |
1da177e4 | 108 | *a |= mask; |
49edd098 | 109 | raw_local_irq_restore(flags); |
1da177e4 LT |
110 | } |
111 | } | |
112 | ||
1da177e4 LT |
113 | /* |
114 | * clear_bit - Clears a bit in memory | |
115 | * @nr: Bit to clear | |
116 | * @addr: Address to start counting from | |
117 | * | |
118 | * clear_bit() is atomic and may not be reordered. However, it does | |
119 | * not contain a memory barrier, so if it is used for locking purposes, | |
120 | * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() | |
121 | * in order to ensure changes are visible on other processors. | |
122 | */ | |
123 | static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) | |
124 | { | |
125 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
b961153b | 126 | unsigned short bit = nr & SZLONG_MASK; |
1da177e4 LT |
127 | unsigned long temp; |
128 | ||
129 | if (cpu_has_llsc && R10000_LLSC_WAR) { | |
130 | __asm__ __volatile__( | |
c4559f67 | 131 | " .set mips3 \n" |
1da177e4 LT |
132 | "1: " __LL "%0, %1 # clear_bit \n" |
133 | " and %0, %2 \n" | |
134 | " " __SC "%0, %1 \n" | |
135 | " beqzl %0, 1b \n" | |
aac8aa77 | 136 | " .set mips0 \n" |
1da177e4 | 137 | : "=&r" (temp), "=m" (*m) |
b961153b | 138 | : "ir" (~(1UL << bit)), "m" (*m)); |
102fa15c | 139 | #ifdef CONFIG_CPU_MIPSR2 |
b961153b | 140 | } else if (__builtin_constant_p(bit)) { |
102fa15c RB |
141 | __asm__ __volatile__( |
142 | "1: " __LL "%0, %1 # clear_bit \n" | |
143 | " " __INS "%0, $0, %2, 1 \n" | |
144 | " " __SC "%0, %1 \n" | |
145 | " beqz %0, 2f \n" | |
146 | " .subsection 2 \n" | |
147 | "2: b 1b \n" | |
148 | " .previous \n" | |
149 | : "=&r" (temp), "=m" (*m) | |
d6d8a463 | 150 | : "ir" (bit), "m" (*m)); |
102fa15c | 151 | #endif /* CONFIG_CPU_MIPSR2 */ |
1da177e4 LT |
152 | } else if (cpu_has_llsc) { |
153 | __asm__ __volatile__( | |
c4559f67 | 154 | " .set mips3 \n" |
1da177e4 LT |
155 | "1: " __LL "%0, %1 # clear_bit \n" |
156 | " and %0, %2 \n" | |
157 | " " __SC "%0, %1 \n" | |
f65e4fa8 RB |
158 | " beqz %0, 2f \n" |
159 | " .subsection 2 \n" | |
160 | "2: b 1b \n" | |
161 | " .previous \n" | |
aac8aa77 | 162 | " .set mips0 \n" |
1da177e4 | 163 | : "=&r" (temp), "=m" (*m) |
b961153b | 164 | : "ir" (~(1UL << bit)), "m" (*m)); |
1da177e4 LT |
165 | } else { |
166 | volatile unsigned long *a = addr; | |
167 | unsigned long mask; | |
4ffd8b38 | 168 | unsigned long flags; |
1da177e4 LT |
169 | |
170 | a += nr >> SZLONG_LOG; | |
b961153b | 171 | mask = 1UL << bit; |
49edd098 | 172 | raw_local_irq_save(flags); |
1da177e4 | 173 | *a &= ~mask; |
49edd098 | 174 | raw_local_irq_restore(flags); |
1da177e4 LT |
175 | } |
176 | } | |
177 | ||
728697cd NP |
178 | /* |
179 | * clear_bit_unlock - Clears a bit in memory | |
180 | * @nr: Bit to clear | |
181 | * @addr: Address to start counting from | |
182 | * | |
183 | * clear_bit() is atomic and implies release semantics before the memory | |
184 | * operation. It can be used for an unlock. | |
185 | */ | |
186 | static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr) | |
187 | { | |
188 | smp_mb__before_clear_bit(); | |
189 | clear_bit(nr, addr); | |
190 | } | |
191 | ||
1da177e4 LT |
192 | /* |
193 | * change_bit - Toggle a bit in memory | |
194 | * @nr: Bit to change | |
195 | * @addr: Address to start counting from | |
196 | * | |
197 | * change_bit() is atomic and may not be reordered. | |
198 | * Note that @nr may be almost arbitrarily large; this function is not | |
199 | * restricted to acting on a single-word quantity. | |
200 | */ | |
201 | static inline void change_bit(unsigned long nr, volatile unsigned long *addr) | |
202 | { | |
b961153b RB |
203 | unsigned short bit = nr & SZLONG_MASK; |
204 | ||
1da177e4 LT |
205 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
206 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
207 | unsigned long temp; | |
208 | ||
209 | __asm__ __volatile__( | |
c4559f67 | 210 | " .set mips3 \n" |
1da177e4 LT |
211 | "1: " __LL "%0, %1 # change_bit \n" |
212 | " xor %0, %2 \n" | |
aac8aa77 | 213 | " " __SC "%0, %1 \n" |
1da177e4 | 214 | " beqzl %0, 1b \n" |
aac8aa77 | 215 | " .set mips0 \n" |
1da177e4 | 216 | : "=&r" (temp), "=m" (*m) |
b961153b | 217 | : "ir" (1UL << bit), "m" (*m)); |
1da177e4 LT |
218 | } else if (cpu_has_llsc) { |
219 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
220 | unsigned long temp; | |
221 | ||
222 | __asm__ __volatile__( | |
c4559f67 | 223 | " .set mips3 \n" |
1da177e4 LT |
224 | "1: " __LL "%0, %1 # change_bit \n" |
225 | " xor %0, %2 \n" | |
aac8aa77 | 226 | " " __SC "%0, %1 \n" |
f65e4fa8 RB |
227 | " beqz %0, 2f \n" |
228 | " .subsection 2 \n" | |
229 | "2: b 1b \n" | |
230 | " .previous \n" | |
aac8aa77 | 231 | " .set mips0 \n" |
1da177e4 | 232 | : "=&r" (temp), "=m" (*m) |
b961153b | 233 | : "ir" (1UL << bit), "m" (*m)); |
1da177e4 LT |
234 | } else { |
235 | volatile unsigned long *a = addr; | |
236 | unsigned long mask; | |
4ffd8b38 | 237 | unsigned long flags; |
1da177e4 LT |
238 | |
239 | a += nr >> SZLONG_LOG; | |
b961153b | 240 | mask = 1UL << bit; |
49edd098 | 241 | raw_local_irq_save(flags); |
1da177e4 | 242 | *a ^= mask; |
49edd098 | 243 | raw_local_irq_restore(flags); |
1da177e4 LT |
244 | } |
245 | } | |
246 | ||
1da177e4 LT |
247 | /* |
248 | * test_and_set_bit - Set a bit and return its old value | |
249 | * @nr: Bit to set | |
250 | * @addr: Address to count from | |
251 | * | |
252 | * This operation is atomic and cannot be reordered. | |
253 | * It also implies a memory barrier. | |
254 | */ | |
255 | static inline int test_and_set_bit(unsigned long nr, | |
256 | volatile unsigned long *addr) | |
257 | { | |
b961153b | 258 | unsigned short bit = nr & SZLONG_MASK; |
ff72b7a6 | 259 | unsigned long res; |
b961153b | 260 | |
c8f30ae5 NP |
261 | smp_llsc_mb(); |
262 | ||
1da177e4 LT |
263 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
264 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
ff72b7a6 | 265 | unsigned long temp; |
1da177e4 LT |
266 | |
267 | __asm__ __volatile__( | |
c4559f67 | 268 | " .set mips3 \n" |
1da177e4 LT |
269 | "1: " __LL "%0, %1 # test_and_set_bit \n" |
270 | " or %2, %0, %3 \n" | |
271 | " " __SC "%2, %1 \n" | |
272 | " beqzl %2, 1b \n" | |
273 | " and %2, %0, %3 \n" | |
aac8aa77 | 274 | " .set mips0 \n" |
1da177e4 | 275 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
b961153b | 276 | : "r" (1UL << bit), "m" (*m) |
1da177e4 | 277 | : "memory"); |
1da177e4 LT |
278 | } else if (cpu_has_llsc) { |
279 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
ff72b7a6 | 280 | unsigned long temp; |
1da177e4 LT |
281 | |
282 | __asm__ __volatile__( | |
aac8aa77 MR |
283 | " .set push \n" |
284 | " .set noreorder \n" | |
c4559f67 | 285 | " .set mips3 \n" |
aac8aa77 | 286 | "1: " __LL "%0, %1 # test_and_set_bit \n" |
1da177e4 LT |
287 | " or %2, %0, %3 \n" |
288 | " " __SC "%2, %1 \n" | |
f65e4fa8 | 289 | " beqz %2, 2f \n" |
1da177e4 | 290 | " and %2, %0, %3 \n" |
f65e4fa8 RB |
291 | " .subsection 2 \n" |
292 | "2: b 1b \n" | |
293 | " nop \n" | |
294 | " .previous \n" | |
aac8aa77 | 295 | " .set pop \n" |
1da177e4 | 296 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
b961153b | 297 | : "r" (1UL << bit), "m" (*m) |
1da177e4 | 298 | : "memory"); |
1da177e4 LT |
299 | } else { |
300 | volatile unsigned long *a = addr; | |
301 | unsigned long mask; | |
4ffd8b38 | 302 | unsigned long flags; |
1da177e4 LT |
303 | |
304 | a += nr >> SZLONG_LOG; | |
b961153b | 305 | mask = 1UL << bit; |
49edd098 | 306 | raw_local_irq_save(flags); |
ff72b7a6 | 307 | res = (mask & *a); |
1da177e4 | 308 | *a |= mask; |
49edd098 | 309 | raw_local_irq_restore(flags); |
1da177e4 | 310 | } |
0004a9df | 311 | |
17099b11 | 312 | smp_llsc_mb(); |
ff72b7a6 RB |
313 | |
314 | return res != 0; | |
1da177e4 LT |
315 | } |
316 | ||
728697cd NP |
317 | /* |
318 | * test_and_set_bit_lock - Set a bit and return its old value | |
319 | * @nr: Bit to set | |
320 | * @addr: Address to count from | |
321 | * | |
322 | * This operation is atomic and implies acquire ordering semantics | |
323 | * after the memory operation. | |
324 | */ | |
325 | static inline int test_and_set_bit_lock(unsigned long nr, | |
326 | volatile unsigned long *addr) | |
327 | { | |
328 | unsigned short bit = nr & SZLONG_MASK; | |
329 | unsigned long res; | |
330 | ||
331 | if (cpu_has_llsc && R10000_LLSC_WAR) { | |
332 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
333 | unsigned long temp; | |
334 | ||
335 | __asm__ __volatile__( | |
336 | " .set mips3 \n" | |
337 | "1: " __LL "%0, %1 # test_and_set_bit \n" | |
338 | " or %2, %0, %3 \n" | |
339 | " " __SC "%2, %1 \n" | |
340 | " beqzl %2, 1b \n" | |
341 | " and %2, %0, %3 \n" | |
342 | " .set mips0 \n" | |
343 | : "=&r" (temp), "=m" (*m), "=&r" (res) | |
344 | : "r" (1UL << bit), "m" (*m) | |
345 | : "memory"); | |
346 | } else if (cpu_has_llsc) { | |
347 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
348 | unsigned long temp; | |
349 | ||
350 | __asm__ __volatile__( | |
351 | " .set push \n" | |
352 | " .set noreorder \n" | |
353 | " .set mips3 \n" | |
354 | "1: " __LL "%0, %1 # test_and_set_bit \n" | |
355 | " or %2, %0, %3 \n" | |
356 | " " __SC "%2, %1 \n" | |
357 | " beqz %2, 2f \n" | |
358 | " and %2, %0, %3 \n" | |
359 | " .subsection 2 \n" | |
360 | "2: b 1b \n" | |
361 | " nop \n" | |
362 | " .previous \n" | |
363 | " .set pop \n" | |
364 | : "=&r" (temp), "=m" (*m), "=&r" (res) | |
365 | : "r" (1UL << bit), "m" (*m) | |
366 | : "memory"); | |
367 | } else { | |
368 | volatile unsigned long *a = addr; | |
369 | unsigned long mask; | |
370 | unsigned long flags; | |
371 | ||
372 | a += nr >> SZLONG_LOG; | |
373 | mask = 1UL << bit; | |
374 | raw_local_irq_save(flags); | |
375 | res = (mask & *a); | |
376 | *a |= mask; | |
377 | raw_local_irq_restore(flags); | |
378 | } | |
379 | ||
380 | smp_llsc_mb(); | |
381 | ||
382 | return res != 0; | |
383 | } | |
1da177e4 LT |
384 | /* |
385 | * test_and_clear_bit - Clear a bit and return its old value | |
386 | * @nr: Bit to clear | |
387 | * @addr: Address to count from | |
388 | * | |
389 | * This operation is atomic and cannot be reordered. | |
390 | * It also implies a memory barrier. | |
391 | */ | |
392 | static inline int test_and_clear_bit(unsigned long nr, | |
393 | volatile unsigned long *addr) | |
394 | { | |
b961153b | 395 | unsigned short bit = nr & SZLONG_MASK; |
ff72b7a6 | 396 | unsigned long res; |
b961153b | 397 | |
c8f30ae5 NP |
398 | smp_llsc_mb(); |
399 | ||
1da177e4 LT |
400 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
401 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
8e09ffb6 | 402 | unsigned long temp; |
1da177e4 LT |
403 | |
404 | __asm__ __volatile__( | |
c4559f67 | 405 | " .set mips3 \n" |
1da177e4 LT |
406 | "1: " __LL "%0, %1 # test_and_clear_bit \n" |
407 | " or %2, %0, %3 \n" | |
408 | " xor %2, %3 \n" | |
aac8aa77 | 409 | " " __SC "%2, %1 \n" |
1da177e4 LT |
410 | " beqzl %2, 1b \n" |
411 | " and %2, %0, %3 \n" | |
aac8aa77 | 412 | " .set mips0 \n" |
1da177e4 | 413 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
b961153b | 414 | : "r" (1UL << bit), "m" (*m) |
1da177e4 | 415 | : "memory"); |
102fa15c RB |
416 | #ifdef CONFIG_CPU_MIPSR2 |
417 | } else if (__builtin_constant_p(nr)) { | |
418 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
ff72b7a6 | 419 | unsigned long temp; |
102fa15c RB |
420 | |
421 | __asm__ __volatile__( | |
422 | "1: " __LL "%0, %1 # test_and_clear_bit \n" | |
423 | " " __EXT "%2, %0, %3, 1 \n" | |
424 | " " __INS "%0, $0, %3, 1 \n" | |
425 | " " __SC "%0, %1 \n" | |
426 | " beqz %0, 2f \n" | |
427 | " .subsection 2 \n" | |
428 | "2: b 1b \n" | |
429 | " .previous \n" | |
430 | : "=&r" (temp), "=m" (*m), "=&r" (res) | |
d6d8a463 | 431 | : "ir" (bit), "m" (*m) |
102fa15c | 432 | : "memory"); |
102fa15c | 433 | #endif |
1da177e4 LT |
434 | } else if (cpu_has_llsc) { |
435 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
ff72b7a6 | 436 | unsigned long temp; |
1da177e4 LT |
437 | |
438 | __asm__ __volatile__( | |
aac8aa77 MR |
439 | " .set push \n" |
440 | " .set noreorder \n" | |
c4559f67 | 441 | " .set mips3 \n" |
aac8aa77 | 442 | "1: " __LL "%0, %1 # test_and_clear_bit \n" |
1da177e4 LT |
443 | " or %2, %0, %3 \n" |
444 | " xor %2, %3 \n" | |
aac8aa77 | 445 | " " __SC "%2, %1 \n" |
f65e4fa8 | 446 | " beqz %2, 2f \n" |
1da177e4 | 447 | " and %2, %0, %3 \n" |
f65e4fa8 RB |
448 | " .subsection 2 \n" |
449 | "2: b 1b \n" | |
450 | " nop \n" | |
451 | " .previous \n" | |
aac8aa77 | 452 | " .set pop \n" |
1da177e4 | 453 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
b961153b | 454 | : "r" (1UL << bit), "m" (*m) |
1da177e4 | 455 | : "memory"); |
1da177e4 LT |
456 | } else { |
457 | volatile unsigned long *a = addr; | |
458 | unsigned long mask; | |
4ffd8b38 | 459 | unsigned long flags; |
1da177e4 LT |
460 | |
461 | a += nr >> SZLONG_LOG; | |
b961153b | 462 | mask = 1UL << bit; |
49edd098 | 463 | raw_local_irq_save(flags); |
ff72b7a6 | 464 | res = (mask & *a); |
1da177e4 | 465 | *a &= ~mask; |
49edd098 | 466 | raw_local_irq_restore(flags); |
1da177e4 | 467 | } |
0004a9df | 468 | |
17099b11 | 469 | smp_llsc_mb(); |
ff72b7a6 RB |
470 | |
471 | return res != 0; | |
1da177e4 LT |
472 | } |
473 | ||
1da177e4 LT |
474 | /* |
475 | * test_and_change_bit - Change a bit and return its old value | |
476 | * @nr: Bit to change | |
477 | * @addr: Address to count from | |
478 | * | |
479 | * This operation is atomic and cannot be reordered. | |
480 | * It also implies a memory barrier. | |
481 | */ | |
482 | static inline int test_and_change_bit(unsigned long nr, | |
483 | volatile unsigned long *addr) | |
484 | { | |
b961153b | 485 | unsigned short bit = nr & SZLONG_MASK; |
ff72b7a6 | 486 | unsigned long res; |
b961153b | 487 | |
c8f30ae5 NP |
488 | smp_llsc_mb(); |
489 | ||
1da177e4 LT |
490 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
491 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
ff72b7a6 | 492 | unsigned long temp; |
1da177e4 LT |
493 | |
494 | __asm__ __volatile__( | |
c4559f67 | 495 | " .set mips3 \n" |
aac8aa77 | 496 | "1: " __LL "%0, %1 # test_and_change_bit \n" |
1da177e4 | 497 | " xor %2, %0, %3 \n" |
aac8aa77 | 498 | " " __SC "%2, %1 \n" |
1da177e4 LT |
499 | " beqzl %2, 1b \n" |
500 | " and %2, %0, %3 \n" | |
aac8aa77 | 501 | " .set mips0 \n" |
1da177e4 | 502 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
b961153b | 503 | : "r" (1UL << bit), "m" (*m) |
1da177e4 | 504 | : "memory"); |
1da177e4 LT |
505 | } else if (cpu_has_llsc) { |
506 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); | |
ff72b7a6 | 507 | unsigned long temp; |
1da177e4 LT |
508 | |
509 | __asm__ __volatile__( | |
aac8aa77 MR |
510 | " .set push \n" |
511 | " .set noreorder \n" | |
c4559f67 | 512 | " .set mips3 \n" |
aac8aa77 | 513 | "1: " __LL "%0, %1 # test_and_change_bit \n" |
1da177e4 | 514 | " xor %2, %0, %3 \n" |
aac8aa77 | 515 | " " __SC "\t%2, %1 \n" |
f65e4fa8 | 516 | " beqz %2, 2f \n" |
1da177e4 | 517 | " and %2, %0, %3 \n" |
f65e4fa8 RB |
518 | " .subsection 2 \n" |
519 | "2: b 1b \n" | |
520 | " nop \n" | |
521 | " .previous \n" | |
aac8aa77 | 522 | " .set pop \n" |
1da177e4 | 523 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
b961153b | 524 | : "r" (1UL << bit), "m" (*m) |
1da177e4 | 525 | : "memory"); |
1da177e4 LT |
526 | } else { |
527 | volatile unsigned long *a = addr; | |
ff72b7a6 | 528 | unsigned long mask; |
4ffd8b38 | 529 | unsigned long flags; |
1da177e4 LT |
530 | |
531 | a += nr >> SZLONG_LOG; | |
b961153b | 532 | mask = 1UL << bit; |
49edd098 | 533 | raw_local_irq_save(flags); |
ff72b7a6 | 534 | res = (mask & *a); |
1da177e4 | 535 | *a ^= mask; |
49edd098 | 536 | raw_local_irq_restore(flags); |
1da177e4 | 537 | } |
0004a9df | 538 | |
17099b11 | 539 | smp_llsc_mb(); |
ff72b7a6 RB |
540 | |
541 | return res != 0; | |
1da177e4 LT |
542 | } |
543 | ||
3c9ee7ef | 544 | #include <asm-generic/bitops/non-atomic.h> |
1da177e4 | 545 | |
728697cd NP |
546 | /* |
547 | * __clear_bit_unlock - Clears a bit in memory | |
548 | * @nr: Bit to clear | |
549 | * @addr: Address to start counting from | |
550 | * | |
551 | * __clear_bit() is non-atomic and implies release semantics before the memory | |
552 | * operation. It can be used for an unlock if no other CPUs can concurrently | |
553 | * modify other bits in the word. | |
554 | */ | |
555 | static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr) | |
556 | { | |
557 | smp_mb(); | |
558 | __clear_bit(nr, addr); | |
559 | } | |
560 | ||
1da177e4 | 561 | /* |
ec917c2c | 562 | * Return the bit position (0..63) of the most significant 1 bit in a word |
65903265 RB |
563 | * Returns -1 if no 1 bit exists |
564 | */ | |
4816227b | 565 | static inline unsigned long __fls(unsigned long word) |
65903265 | 566 | { |
4816227b | 567 | int num; |
65903265 | 568 | |
4816227b RB |
569 | if (BITS_PER_LONG == 32 && |
570 | __builtin_constant_p(cpu_has_mips_r) && cpu_has_mips_r) { | |
49a89efb | 571 | __asm__( |
ec917c2c RB |
572 | " .set push \n" |
573 | " .set mips32 \n" | |
574 | " clz %0, %1 \n" | |
575 | " .set pop \n" | |
4816227b RB |
576 | : "=r" (num) |
577 | : "r" (word)); | |
65903265 | 578 | |
4816227b | 579 | return 31 - num; |
ec917c2c RB |
580 | } |
581 | ||
4816227b RB |
582 | if (BITS_PER_LONG == 64 && |
583 | __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) { | |
584 | __asm__( | |
585 | " .set push \n" | |
586 | " .set mips64 \n" | |
587 | " dclz %0, %1 \n" | |
588 | " .set pop \n" | |
589 | : "=r" (num) | |
590 | : "r" (word)); | |
65903265 | 591 | |
4816227b RB |
592 | return 63 - num; |
593 | } | |
594 | ||
595 | num = BITS_PER_LONG - 1; | |
65903265 | 596 | |
4816227b RB |
597 | #if BITS_PER_LONG == 64 |
598 | if (!(word & (~0ul << 32))) { | |
599 | num -= 32; | |
600 | word <<= 32; | |
601 | } | |
602 | #endif | |
603 | if (!(word & (~0ul << (BITS_PER_LONG-16)))) { | |
604 | num -= 16; | |
605 | word <<= 16; | |
606 | } | |
607 | if (!(word & (~0ul << (BITS_PER_LONG-8)))) { | |
608 | num -= 8; | |
609 | word <<= 8; | |
610 | } | |
611 | if (!(word & (~0ul << (BITS_PER_LONG-4)))) { | |
612 | num -= 4; | |
613 | word <<= 4; | |
614 | } | |
615 | if (!(word & (~0ul << (BITS_PER_LONG-2)))) { | |
616 | num -= 2; | |
617 | word <<= 2; | |
618 | } | |
619 | if (!(word & (~0ul << (BITS_PER_LONG-1)))) | |
620 | num -= 1; | |
621 | return num; | |
65903265 | 622 | } |
65903265 RB |
623 | |
624 | /* | |
625 | * __ffs - find first bit in word. | |
1da177e4 LT |
626 | * @word: The word to search |
627 | * | |
65903265 RB |
628 | * Returns 0..SZLONG-1 |
629 | * Undefined if no bit exists, so code should check against 0 first. | |
1da177e4 | 630 | */ |
65903265 | 631 | static inline unsigned long __ffs(unsigned long word) |
1da177e4 | 632 | { |
ddc0d009 | 633 | return __fls(word & -word); |
1da177e4 LT |
634 | } |
635 | ||
636 | /* | |
bc818247 | 637 | * fls - find last bit set. |
1da177e4 LT |
638 | * @word: The word to search |
639 | * | |
bc818247 AN |
640 | * This is defined the same way as ffs. |
641 | * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. | |
1da177e4 | 642 | */ |
4816227b | 643 | static inline int fls(int x) |
1da177e4 | 644 | { |
4816227b | 645 | int r; |
65903265 | 646 | |
4816227b RB |
647 | if (__builtin_constant_p(cpu_has_mips_r) && cpu_has_mips_r) { |
648 | __asm__("clz %0, %1" : "=r" (x) : "r" (x)); | |
1da177e4 | 649 | |
4816227b RB |
650 | return 32 - x; |
651 | } | |
bc818247 | 652 | |
4816227b RB |
653 | r = 32; |
654 | if (!x) | |
655 | return 0; | |
656 | if (!(x & 0xffff0000u)) { | |
657 | x <<= 16; | |
658 | r -= 16; | |
659 | } | |
660 | if (!(x & 0xff000000u)) { | |
661 | x <<= 8; | |
662 | r -= 8; | |
663 | } | |
664 | if (!(x & 0xf0000000u)) { | |
665 | x <<= 4; | |
666 | r -= 4; | |
667 | } | |
668 | if (!(x & 0xc0000000u)) { | |
669 | x <<= 2; | |
670 | r -= 2; | |
671 | } | |
672 | if (!(x & 0x80000000u)) { | |
673 | x <<= 1; | |
674 | r -= 1; | |
675 | } | |
676 | return r; | |
65903265 | 677 | } |
4816227b | 678 | |
bc818247 | 679 | #include <asm-generic/bitops/fls64.h> |
65903265 RB |
680 | |
681 | /* | |
bc818247 | 682 | * ffs - find first bit set. |
65903265 RB |
683 | * @word: The word to search |
684 | * | |
bc818247 AN |
685 | * This is defined the same way as |
686 | * the libc and compiler builtin ffs routines, therefore | |
687 | * differs in spirit from the above ffz (man ffs). | |
65903265 | 688 | */ |
bc818247 | 689 | static inline int ffs(int word) |
65903265 | 690 | { |
bc818247 AN |
691 | if (!word) |
692 | return 0; | |
2caf1900 | 693 | |
bc818247 | 694 | return fls(word & -word); |
65903265 RB |
695 | } |
696 | ||
bc818247 | 697 | #include <asm-generic/bitops/ffz.h> |
3c9ee7ef | 698 | #include <asm-generic/bitops/find.h> |
1da177e4 LT |
699 | |
700 | #ifdef __KERNEL__ | |
701 | ||
3c9ee7ef AM |
702 | #include <asm-generic/bitops/sched.h> |
703 | #include <asm-generic/bitops/hweight.h> | |
704 | #include <asm-generic/bitops/ext2-non-atomic.h> | |
705 | #include <asm-generic/bitops/ext2-atomic.h> | |
706 | #include <asm-generic/bitops/minix.h> | |
1da177e4 LT |
707 | |
708 | #endif /* __KERNEL__ */ | |
709 | ||
710 | #endif /* _ASM_BITOPS_H */ |