Commit | Line | Data |
---|---|---|
fef74705 RB |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * Copyright (C) 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) | |
7 | */ | |
8 | #ifndef __ASM_CMPXCHG_H | |
9 | #define __ASM_CMPXCHG_H | |
10 | ||
5520e426 | 11 | #include <linux/bug.h> |
fef74705 | 12 | #include <linux/irqflags.h> |
b0984c43 | 13 | #include <asm/compiler.h> |
b81947c6 DH |
14 | #include <asm/war.h> |
15 | ||
6b1e7629 PB |
16 | /* |
17 | * Using a branch-likely instruction to check the result of an sc instruction | |
18 | * works around a bug present in R10000 CPUs prior to revision 3.0 that could | |
19 | * cause ll-sc sequences to execute non-atomically. | |
20 | */ | |
21 | #if R10000_LLSC_WAR | |
22 | # define __scbeqz "beqzl" | |
23 | #else | |
24 | # define __scbeqz "beqz" | |
25 | #endif | |
26 | ||
d15dc68c PB |
27 | /* |
28 | * These functions doesn't exist, so if they are called you'll either: | |
29 | * | |
30 | * - Get an error at compile-time due to __compiletime_error, if supported by | |
31 | * your compiler. | |
32 | * | |
33 | * or: | |
34 | * | |
35 | * - Get an error at link-time due to the call to the missing function. | |
36 | */ | |
8263db4d | 37 | extern unsigned long __cmpxchg_called_with_bad_pointer(void) |
d15dc68c PB |
38 | __compiletime_error("Bad argument size for cmpxchg"); |
39 | extern unsigned long __xchg_called_with_bad_pointer(void) | |
40 | __compiletime_error("Bad argument size for xchg"); | |
41 | ||
5154f3b4 PB |
42 | #define __xchg_asm(ld, st, m, val) \ |
43 | ({ \ | |
44 | __typeof(*(m)) __ret; \ | |
45 | \ | |
46 | if (kernel_uses_llsc) { \ | |
47 | __asm__ __volatile__( \ | |
48 | " .set push \n" \ | |
49 | " .set noat \n" \ | |
50 | " .set " MIPS_ISA_ARCH_LEVEL " \n" \ | |
51 | "1: " ld " %0, %2 # __xchg_asm \n" \ | |
52 | " .set mips0 \n" \ | |
53 | " move $1, %z3 \n" \ | |
54 | " .set " MIPS_ISA_ARCH_LEVEL " \n" \ | |
55 | " " st " $1, %1 \n" \ | |
56 | "\t" __scbeqz " $1, 1b \n" \ | |
57 | " .set pop \n" \ | |
58 | : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ | |
59 | : GCC_OFF_SMALL_ASM() (*m), "Jr" (val) \ | |
60 | : "memory"); \ | |
61 | } else { \ | |
62 | unsigned long __flags; \ | |
63 | \ | |
64 | raw_local_irq_save(__flags); \ | |
65 | __ret = *m; \ | |
66 | *m = val; \ | |
67 | raw_local_irq_restore(__flags); \ | |
68 | } \ | |
69 | \ | |
70 | __ret; \ | |
71 | }) | |
72 | ||
b70eb300 PB |
73 | extern unsigned long __xchg_small(volatile void *ptr, unsigned long val, |
74 | unsigned int size); | |
75 | ||
b81947c6 DH |
76 | static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) |
77 | { | |
78 | switch (size) { | |
b70eb300 PB |
79 | case 1: |
80 | case 2: | |
81 | return __xchg_small(ptr, x, size); | |
82 | ||
b81947c6 | 83 | case 4: |
62c6081d PB |
84 | return __xchg_asm("ll", "sc", (volatile u32 *)ptr, x); |
85 | ||
b81947c6 | 86 | case 8: |
62c6081d PB |
87 | if (!IS_ENABLED(CONFIG_64BIT)) |
88 | return __xchg_called_with_bad_pointer(); | |
89 | ||
90 | return __xchg_asm("lld", "scd", (volatile u64 *)ptr, x); | |
91 | ||
d15dc68c PB |
92 | default: |
93 | return __xchg_called_with_bad_pointer(); | |
b81947c6 | 94 | } |
b81947c6 DH |
95 | } |
96 | ||
97 | #define xchg(ptr, x) \ | |
98 | ({ \ | |
62c6081d PB |
99 | __typeof__(*(ptr)) __res; \ |
100 | \ | |
62c6081d PB |
101 | smp_mb__before_llsc(); \ |
102 | \ | |
103 | __res = (__typeof__(*(ptr))) \ | |
104 | __xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \ | |
105 | \ | |
106 | smp_llsc_mb(); \ | |
107 | \ | |
108 | __res; \ | |
b81947c6 | 109 | }) |
fef74705 | 110 | |
fef74705 RB |
111 | #define __cmpxchg_asm(ld, st, m, old, new) \ |
112 | ({ \ | |
113 | __typeof(*(m)) __ret; \ | |
114 | \ | |
6b1e7629 | 115 | if (kernel_uses_llsc) { \ |
fef74705 RB |
116 | __asm__ __volatile__( \ |
117 | " .set push \n" \ | |
118 | " .set noat \n" \ | |
fa998ebb | 119 | " .set "MIPS_ISA_ARCH_LEVEL" \n" \ |
70342287 | 120 | "1: " ld " %0, %2 # __cmpxchg_asm \n" \ |
fef74705 RB |
121 | " bne %0, %z3, 2f \n" \ |
122 | " .set mips0 \n" \ | |
123 | " move $1, %z4 \n" \ | |
fa998ebb | 124 | " .set "MIPS_ISA_ARCH_LEVEL" \n" \ |
fef74705 | 125 | " " st " $1, %1 \n" \ |
6b1e7629 | 126 | "\t" __scbeqz " $1, 1b \n" \ |
fef74705 | 127 | " .set pop \n" \ |
7837314d | 128 | "2: \n" \ |
94bfb75a MC |
129 | : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ |
130 | : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \ | |
fef74705 RB |
131 | : "memory"); \ |
132 | } else { \ | |
133 | unsigned long __flags; \ | |
134 | \ | |
135 | raw_local_irq_save(__flags); \ | |
136 | __ret = *m; \ | |
137 | if (__ret == old) \ | |
138 | *m = new; \ | |
139 | raw_local_irq_restore(__flags); \ | |
140 | } \ | |
141 | \ | |
142 | __ret; \ | |
143 | }) | |
144 | ||
3ba7f44d PB |
145 | extern unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old, |
146 | unsigned long new, unsigned int size); | |
147 | ||
8263db4d PB |
148 | static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, |
149 | unsigned long new, unsigned int size) | |
150 | { | |
151 | switch (size) { | |
3ba7f44d PB |
152 | case 1: |
153 | case 2: | |
154 | return __cmpxchg_small(ptr, old, new, size); | |
155 | ||
8263db4d PB |
156 | case 4: |
157 | return __cmpxchg_asm("ll", "sc", (volatile u32 *)ptr, old, new); | |
158 | ||
159 | case 8: | |
160 | /* lld/scd are only available for MIPS64 */ | |
161 | if (!IS_ENABLED(CONFIG_64BIT)) | |
162 | return __cmpxchg_called_with_bad_pointer(); | |
163 | ||
164 | return __cmpxchg_asm("lld", "scd", (volatile u64 *)ptr, old, new); | |
165 | ||
166 | default: | |
167 | return __cmpxchg_called_with_bad_pointer(); | |
168 | } | |
169 | } | |
170 | ||
171 | #define cmpxchg_local(ptr, old, new) \ | |
172 | ((__typeof__(*(ptr))) \ | |
173 | __cmpxchg((ptr), \ | |
174 | (unsigned long)(__typeof__(*(ptr)))(old), \ | |
175 | (unsigned long)(__typeof__(*(ptr)))(new), \ | |
176 | sizeof(*(ptr)))) | |
177 | ||
178 | #define cmpxchg(ptr, old, new) \ | |
fef74705 | 179 | ({ \ |
8263db4d | 180 | __typeof__(*(ptr)) __res; \ |
fef74705 | 181 | \ |
8263db4d PB |
182 | smp_mb__before_llsc(); \ |
183 | __res = cmpxchg_local((ptr), (old), (new)); \ | |
184 | smp_llsc_mb(); \ | |
fef74705 RB |
185 | \ |
186 | __res; \ | |
187 | }) | |
188 | ||
e2093c7b DCZ |
189 | #ifdef CONFIG_64BIT |
190 | #define cmpxchg64_local(ptr, o, n) \ | |
3b96a56d MD |
191 | ({ \ |
192 | BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ | |
e2093c7b | 193 | cmpxchg_local((ptr), (o), (n)); \ |
3b96a56d MD |
194 | }) |
195 | ||
e2093c7b | 196 | #define cmpxchg64(ptr, o, n) \ |
3b96a56d MD |
197 | ({ \ |
198 | BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ | |
e2093c7b | 199 | cmpxchg((ptr), (o), (n)); \ |
3b96a56d MD |
200 | }) |
201 | #else | |
202 | #include <asm-generic/cmpxchg-local.h> | |
203 | #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) | |
e2093c7b | 204 | #define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) |
3b96a56d MD |
205 | #endif |
206 | ||
6b1e7629 PB |
207 | #undef __scbeqz |
208 | ||
fef74705 | 209 | #endif /* __ASM_CMPXCHG_H */ |