Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1a3b1d89 BG |
2 | #ifndef _ASM_X86_ATOMIC64_32_H |
3 | #define _ASM_X86_ATOMIC64_32_H | |
4 | ||
5 | #include <linux/compiler.h> | |
6 | #include <linux/types.h> | |
1a3b1d89 BG |
7 | //#include <asm/cmpxchg.h> |
8 | ||
9 | /* An 64bit atomic type */ | |
10 | ||
11 | typedef struct { | |
79c53a83 | 12 | s64 __aligned(8) counter; |
1a3b1d89 BG |
13 | } atomic64_t; |
14 | ||
15 | #define ATOMIC64_INIT(val) { (val) } | |
16 | ||
e73c4e34 UB |
17 | /* |
18 | * Read an atomic64_t non-atomically. | |
19 | * | |
20 | * This is intended to be used in cases where a subsequent atomic operation | |
21 | * will handle the torn value, and can be used to prime the first iteration | |
22 | * of unconditional try_cmpxchg() loops, e.g.: | |
23 | * | |
24 | * s64 val = arch_atomic64_read_nonatomic(v); | |
25 | * do { } while (!arch_atomic64_try_cmpxchg(v, &val, val OP i); | |
26 | * | |
27 | * This is NOT safe to use where the value is not always checked by a | |
28 | * subsequent atomic operation, such as in conditional try_cmpxchg() loops | |
29 | * that can break before the atomic operation, e.g.: | |
30 | * | |
31 | * s64 val = arch_atomic64_read_nonatomic(v); | |
32 | * do { | |
33 | * if (condition(val)) | |
34 | * break; | |
35 | * } while (!arch_atomic64_try_cmpxchg(v, &val, val OP i); | |
36 | */ | |
37 | static __always_inline s64 arch_atomic64_read_nonatomic(const atomic64_t *v) | |
38 | { | |
39 | /* See comment in arch_atomic_read(). */ | |
40 | return __READ_ONCE(v->counter); | |
41 | } | |
42 | ||
819165fb JB |
43 | #define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...) |
44 | #ifndef ATOMIC64_EXPORT | |
45 | #define ATOMIC64_DECL_ONE __ATOMIC64_DECL | |
46 | #else | |
47 | #define ATOMIC64_DECL_ONE(sym) __ATOMIC64_DECL(sym); \ | |
48 | ATOMIC64_EXPORT(atomic64_##sym) | |
49 | #endif | |
50 | ||
a7e926ab | 51 | #ifdef CONFIG_X86_CMPXCHG64 |
819165fb | 52 | #define __alternative_atomic64(f, g, out, in...) \ |
41cd2e1e | 53 | asm volatile("call %c[func]" \ |
819165fb JB |
54 | : out : [func] "i" (atomic64_##g##_cx8), ## in) |
55 | ||
56 | #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8) | |
a7e926ab | 57 | #else |
819165fb JB |
58 | #define __alternative_atomic64(f, g, out, in...) \ |
59 | alternative_call(atomic64_##f##_386, atomic64_##g##_cx8, \ | |
60 | X86_FEATURE_CX8, ASM_OUTPUT2(out), ## in) | |
61 | ||
62 | #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \ | |
63 | ATOMIC64_DECL_ONE(sym##_386) | |
64 | ||
65 | ATOMIC64_DECL_ONE(add_386); | |
66 | ATOMIC64_DECL_ONE(sub_386); | |
67 | ATOMIC64_DECL_ONE(inc_386); | |
68 | ATOMIC64_DECL_ONE(dec_386); | |
a7e926ab LB |
69 | #endif |
70 | ||
819165fb JB |
71 | #define alternative_atomic64(f, out, in...) \ |
72 | __alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in) | |
73 | ||
74 | ATOMIC64_DECL(read); | |
75 | ATOMIC64_DECL(set); | |
76 | ATOMIC64_DECL(xchg); | |
77 | ATOMIC64_DECL(add_return); | |
78 | ATOMIC64_DECL(sub_return); | |
79 | ATOMIC64_DECL(inc_return); | |
80 | ATOMIC64_DECL(dec_return); | |
81 | ATOMIC64_DECL(dec_if_positive); | |
82 | ATOMIC64_DECL(inc_not_zero); | |
83 | ATOMIC64_DECL(add_unless); | |
84 | ||
85 | #undef ATOMIC64_DECL | |
86 | #undef ATOMIC64_DECL_ONE | |
87 | #undef __ATOMIC64_DECL | |
88 | #undef ATOMIC64_EXPORT | |
a7e926ab | 89 | |
276b8930 | 90 | static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) |
a7e926ab | 91 | { |
276b8930 | 92 | return arch_cmpxchg64(&v->counter, old, new); |
a7e926ab | 93 | } |
37f8173d | 94 | #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg |
1a3b1d89 | 95 | |
276b8930 UB |
96 | static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) |
97 | { | |
98 | return arch_try_cmpxchg64(&v->counter, old, new); | |
99 | } | |
100 | #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg | |
101 | ||
7aab7aa4 | 102 | static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n) |
a7e926ab | 103 | { |
79c53a83 | 104 | s64 o; |
a7e926ab LB |
105 | unsigned high = (unsigned)(n >> 32); |
106 | unsigned low = (unsigned)n; | |
819165fb JB |
107 | alternative_atomic64(xchg, "=&A" (o), |
108 | "S" (v), "b" (low), "c" (high) | |
109 | : "memory"); | |
a7e926ab LB |
110 | return o; |
111 | } | |
37f8173d | 112 | #define arch_atomic64_xchg arch_atomic64_xchg |
1a3b1d89 | 113 | |
7aab7aa4 | 114 | static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) |
a7e926ab LB |
115 | { |
116 | unsigned high = (unsigned)(i >> 32); | |
117 | unsigned low = (unsigned)i; | |
819165fb JB |
118 | alternative_atomic64(set, /* no output */, |
119 | "S" (v), "b" (low), "c" (high) | |
120 | : "eax", "edx", "memory"); | |
a7e926ab | 121 | } |
1a3b1d89 | 122 | |
7aab7aa4 | 123 | static __always_inline s64 arch_atomic64_read(const atomic64_t *v) |
1a3b1d89 | 124 | { |
79c53a83 | 125 | s64 r; |
819165fb | 126 | alternative_atomic64(read, "=&A" (r), "c" (v) : "memory"); |
a7e926ab | 127 | return r; |
447a5647 | 128 | } |
1a3b1d89 | 129 | |
7aab7aa4 | 130 | static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) |
a7e926ab | 131 | { |
819165fb JB |
132 | alternative_atomic64(add_return, |
133 | ASM_OUTPUT2("+A" (i), "+c" (v)), | |
134 | ASM_NO_INPUT_CLOBBER("memory")); | |
a7e926ab LB |
135 | return i; |
136 | } | |
37f8173d | 137 | #define arch_atomic64_add_return arch_atomic64_add_return |
1a3b1d89 | 138 | |
7aab7aa4 | 139 | static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v) |
a7e926ab | 140 | { |
819165fb JB |
141 | alternative_atomic64(sub_return, |
142 | ASM_OUTPUT2("+A" (i), "+c" (v)), | |
143 | ASM_NO_INPUT_CLOBBER("memory")); | |
a7e926ab LB |
144 | return i; |
145 | } | |
37f8173d | 146 | #define arch_atomic64_sub_return arch_atomic64_sub_return |
a7e926ab | 147 | |
7aab7aa4 | 148 | static __always_inline s64 arch_atomic64_inc_return(atomic64_t *v) |
a7e926ab | 149 | { |
79c53a83 | 150 | s64 a; |
819165fb JB |
151 | alternative_atomic64(inc_return, "=&A" (a), |
152 | "S" (v) : "memory", "ecx"); | |
a7e926ab LB |
153 | return a; |
154 | } | |
9837559d | 155 | #define arch_atomic64_inc_return arch_atomic64_inc_return |
a7e926ab | 156 | |
7aab7aa4 | 157 | static __always_inline s64 arch_atomic64_dec_return(atomic64_t *v) |
a7e926ab | 158 | { |
79c53a83 | 159 | s64 a; |
819165fb JB |
160 | alternative_atomic64(dec_return, "=&A" (a), |
161 | "S" (v) : "memory", "ecx"); | |
a7e926ab LB |
162 | return a; |
163 | } | |
9837559d | 164 | #define arch_atomic64_dec_return arch_atomic64_dec_return |
1a3b1d89 | 165 | |
dce2a224 | 166 | static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v) |
a7e926ab | 167 | { |
819165fb JB |
168 | __alternative_atomic64(add, add_return, |
169 | ASM_OUTPUT2("+A" (i), "+c" (v)), | |
170 | ASM_NO_INPUT_CLOBBER("memory")); | |
a7e926ab | 171 | } |
1a3b1d89 | 172 | |
dce2a224 | 173 | static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v) |
a7e926ab | 174 | { |
819165fb JB |
175 | __alternative_atomic64(sub, sub_return, |
176 | ASM_OUTPUT2("+A" (i), "+c" (v)), | |
177 | ASM_NO_INPUT_CLOBBER("memory")); | |
a7e926ab | 178 | } |
1a3b1d89 | 179 | |
7aab7aa4 | 180 | static __always_inline void arch_atomic64_inc(atomic64_t *v) |
a7e926ab | 181 | { |
819165fb JB |
182 | __alternative_atomic64(inc, inc_return, /* no output */, |
183 | "S" (v) : "memory", "eax", "ecx", "edx"); | |
a7e926ab | 184 | } |
4331f4d5 | 185 | #define arch_atomic64_inc arch_atomic64_inc |
1a3b1d89 | 186 | |
7aab7aa4 | 187 | static __always_inline void arch_atomic64_dec(atomic64_t *v) |
a7e926ab | 188 | { |
819165fb JB |
189 | __alternative_atomic64(dec, dec_return, /* no output */, |
190 | "S" (v) : "memory", "eax", "ecx", "edx"); | |
a7e926ab | 191 | } |
4331f4d5 | 192 | #define arch_atomic64_dec arch_atomic64_dec |
1a3b1d89 | 193 | |
7aab7aa4 | 194 | static __always_inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) |
a7e926ab LB |
195 | { |
196 | unsigned low = (unsigned)u; | |
197 | unsigned high = (unsigned)(u >> 32); | |
819165fb | 198 | alternative_atomic64(add_unless, |
cb8095bb JB |
199 | ASM_OUTPUT2("+A" (a), "+c" (low), "+D" (high)), |
200 | "S" (v) : "memory"); | |
a7e926ab LB |
201 | return (int)a; |
202 | } | |
37f8173d | 203 | #define arch_atomic64_add_unless arch_atomic64_add_unless |
a7e926ab | 204 | |
7aab7aa4 | 205 | static __always_inline int arch_atomic64_inc_not_zero(atomic64_t *v) |
a7e926ab LB |
206 | { |
207 | int r; | |
819165fb JB |
208 | alternative_atomic64(inc_not_zero, "=&a" (r), |
209 | "S" (v) : "ecx", "edx", "memory"); | |
a7e926ab LB |
210 | return r; |
211 | } | |
4331f4d5 | 212 | #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero |
a7e926ab | 213 | |
7aab7aa4 | 214 | static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) |
a7e926ab | 215 | { |
79c53a83 | 216 | s64 r; |
819165fb JB |
217 | alternative_atomic64(dec_if_positive, "=&A" (r), |
218 | "S" (v) : "ecx", "memory"); | |
a7e926ab LB |
219 | return r; |
220 | } | |
4331f4d5 | 221 | #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive |
a7e926ab | 222 | |
819165fb JB |
223 | #undef alternative_atomic64 |
224 | #undef __alternative_atomic64 | |
1a3b1d89 | 225 | |
7aab7aa4 | 226 | static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v) |
ba1c9f83 | 227 | { |
95ece481 | 228 | s64 val = arch_atomic64_read_nonatomic(v); |
ba1c9f83 | 229 | |
95ece481 | 230 | do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i)); |
7fc1845d PZ |
231 | } |
232 | ||
7aab7aa4 | 233 | static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) |
ba1c9f83 | 234 | { |
95ece481 | 235 | s64 val = arch_atomic64_read_nonatomic(v); |
ba1c9f83 | 236 | |
95ece481 | 237 | do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i)); |
ba1c9f83 | 238 | |
95ece481 | 239 | return val; |
a8bcccab | 240 | } |
37f8173d | 241 | #define arch_atomic64_fetch_and arch_atomic64_fetch_and |
a8bcccab | 242 | |
7aab7aa4 | 243 | static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v) |
ba1c9f83 | 244 | { |
95ece481 | 245 | s64 val = arch_atomic64_read_nonatomic(v); |
a8bcccab | 246 | |
95ece481 | 247 | do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i)); |
ba1c9f83 DV |
248 | } |
249 | ||
7aab7aa4 | 250 | static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) |
ba1c9f83 | 251 | { |
95ece481 | 252 | s64 val = arch_atomic64_read_nonatomic(v); |
ba1c9f83 | 253 | |
95ece481 | 254 | do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i)); |
ba1c9f83 | 255 | |
95ece481 | 256 | return val; |
ba1c9f83 | 257 | } |
37f8173d | 258 | #define arch_atomic64_fetch_or arch_atomic64_fetch_or |
a8bcccab | 259 | |
7aab7aa4 | 260 | static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v) |
ba1c9f83 | 261 | { |
95ece481 | 262 | s64 val = arch_atomic64_read_nonatomic(v); |
ba1c9f83 | 263 | |
95ece481 | 264 | do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i)); |
ba1c9f83 | 265 | } |
a8bcccab | 266 | |
7aab7aa4 | 267 | static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) |
ba1c9f83 | 268 | { |
95ece481 | 269 | s64 val = arch_atomic64_read_nonatomic(v); |
ba1c9f83 | 270 | |
95ece481 | 271 | do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i)); |
ba1c9f83 | 272 | |
95ece481 | 273 | return val; |
ba1c9f83 | 274 | } |
37f8173d | 275 | #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor |
7fc1845d | 276 | |
7aab7aa4 | 277 | static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) |
ba1c9f83 | 278 | { |
95ece481 | 279 | s64 val = arch_atomic64_read_nonatomic(v); |
ba1c9f83 | 280 | |
95ece481 | 281 | do { } while (!arch_atomic64_try_cmpxchg(v, &val, val + i)); |
ba1c9f83 | 282 | |
95ece481 | 283 | return val; |
ba1c9f83 | 284 | } |
37f8173d | 285 | #define arch_atomic64_fetch_add arch_atomic64_fetch_add |
ba1c9f83 | 286 | |
8bf705d1 | 287 | #define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), (v)) |
7fc1845d | 288 | |
1a3b1d89 | 289 | #endif /* _ASM_X86_ATOMIC64_32_H */ |