projects / linux-2.6-block.git / blame
| Commit | Line | Data |
|---|---|---|
| b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
| 1da177e4 LT |
2 | #ifndef _ASM_GENERIC_PERCPU_H_ |
| 3 | #define _ASM_GENERIC_PERCPU_H_ | |
| 5028eaa9 | 4 | |
| 1da177e4 | 5 | #include <linux/compiler.h> |
| ae1ee11b | 6 | #include <linux/threads.h> |
| 5028eaa9 | 7 | #include <linux/percpu-defs.h> |
| acdac872 | 8 | |
| 1da177e4 LT |
9 | #ifdef CONFIG_SMP |
| 10 | ||
| acdac872 | 11 | /* |
| 12 | * per_cpu_offset() is the offset that has to be added to a | |
| 13 | * percpu variable to get to the instance for a certain processor. | |
| 14 | * | |
| 15 | * Most arches use the __per_cpu_offset array for those offsets but | |
| 16 | * some arches have their own ways of determining the offset (x86_64, s390). | |
| 17 | */ | |
| 18 | #ifndef __per_cpu_offset | |
| 1da177e4 LT |
19 | extern unsigned long __per_cpu_offset[NR_CPUS]; |
| 20 | ||
| a875a69f | 21 | #define per_cpu_offset(x) (__per_cpu_offset[x]) |
| acdac872 | 22 | #endif |
| 23 | ||
| 24 | /* | |
| 25 | * Determine the offset for the currently active processor. | |
| 26 | * An arch may define __my_cpu_offset to provide a more effective | |
| 27 | * means of obtaining the offset to the per cpu variables of the | |
| 28 | * current processor. | |
| 29 | */ | |
| 30 | #ifndef __my_cpu_offset | |
| 31 | #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id()) | |
| 1e835278 HD |
32 | #endif |
| 33 | #ifdef CONFIG_DEBUG_PREEMPT | |
| acdac872 | 34 | #define my_cpu_offset per_cpu_offset(smp_processor_id()) |
| 35 | #else | |
| 36 | #define my_cpu_offset __my_cpu_offset | |
| 37 | #endif | |
| 38 | ||
| bbc344e1 TH |
39 | /* |
| 40 | * Arch may define arch_raw_cpu_ptr() to provide more efficient address | |
| 41 | * translations for raw_cpu_ptr(). | |
| 42 | */ | |
| 43 | #ifndef arch_raw_cpu_ptr | |
| 44 | #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset) | |
| db7829c6 | 45 | #endif |
| bbc344e1 | 46 | |
| dd5af90a | 47 | #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA |
| acdac872 | 48 | extern void setup_per_cpu_areas(void); |
| 49 | #endif | |
| 1da177e4 | 50 | |
| 1da177e4 LT |
51 | #endif /* SMP */ |
| 52 | ||
| 9b8de747 DH |
53 | #ifndef PER_CPU_BASE_SECTION |
| 54 | #ifdef CONFIG_SMP | |
| 3d9a854c | 55 | #define PER_CPU_BASE_SECTION ".data..percpu" |
| 9b8de747 DH |
56 | #else |
| 57 | #define PER_CPU_BASE_SECTION ".data" | |
| 58 | #endif | |
| 59 | #endif | |
| 60 | ||
| acdac872 | 61 | #ifndef PER_CPU_ATTRIBUTES |
| 62 | #define PER_CPU_ATTRIBUTES | |
| 63 | #endif | |
| 64 | ||
| 1b5ca121 NP |
65 | #define raw_cpu_generic_read(pcp) \ |
| 66 | ({ \ | |
| 67 | *raw_cpu_ptr(&(pcp)); \ | |
| 68 | }) | |
| 69 | ||
| 9c28278a TH |
70 | #define raw_cpu_generic_to_op(pcp, val, op) \ |
| 71 | do { \ | |
| 72 | *raw_cpu_ptr(&(pcp)) op val; \ | |
| 73 | } while (0) | |
| 74 | ||
| 75 | #define raw_cpu_generic_add_return(pcp, val) \ | |
| 76 | ({ \ | |
| becef184 | 77 | typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ |
| 1b5ca121 NP |
78 | \ |
| 79 | *__p += val; \ | |
| 80 | *__p; \ | |
| 9c28278a TH |
81 | }) |
| 82 | ||
| 83 | #define raw_cpu_generic_xchg(pcp, nval) \ | |
| eba11788 | 84 | ({ \ |
| becef184 | 85 | typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ |
| eba11788 | 86 | typeof(pcp) __ret; \ |
| 1b5ca121 NP |
87 | __ret = *__p; \ |
| 88 | *__p = nval; \ | |
| eba11788 | 89 | __ret; \ |
| 9c28278a TH |
90 | }) |
| 91 | ||
| c5c0ba95 PZ |
92 | #define __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, _cmpxchg) \ |
| 93 | ({ \ | |
| 94 | typeof(pcp) __val, __old = *(ovalp); \ | |
| 95 | __val = _cmpxchg(pcp, __old, nval); \ | |
| 96 | if (__val != __old) \ | |
| 97 | *(ovalp) = __val; \ | |
| 98 | __val == __old; \ | |
| 99 | }) | |
| 100 | ||
| 101 | #define raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \ | |
| 9c28278a | 102 | ({ \ |
| becef184 | 103 | typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ |
| 093d9b24 | 104 | typeof(pcp) __val = *__p, ___old = *(ovalp); \ |
| c5c0ba95 | 105 | bool __ret; \ |
| 093d9b24 | 106 | if (__val == ___old) { \ |
| 1b5ca121 | 107 | *__p = nval; \ |
| c5c0ba95 PZ |
108 | __ret = true; \ |
| 109 | } else { \ | |
| 110 | *(ovalp) = __val; \ | |
| 111 | __ret = false; \ | |
| 112 | } \ | |
| eba11788 | 113 | __ret; \ |
| 9c28278a TH |
114 | }) |
| 115 | ||
| c5c0ba95 PZ |
116 | #define raw_cpu_generic_cmpxchg(pcp, oval, nval) \ |
| 117 | ({ \ | |
| 118 | typeof(pcp) __old = (oval); \ | |
| 119 | raw_cpu_generic_try_cmpxchg(pcp, &__old, nval); \ | |
| 120 | __old; \ | |
| 121 | }) | |
| 122 | ||
| e88d62cd | 123 | #define __this_cpu_generic_read_nopreempt(pcp) \ |
| eba11788 | 124 | ({ \ |
| 29813a22 | 125 | typeof(pcp) ___ret; \ |
| 7f8d61f0 | 126 | preempt_disable_notrace(); \ |
| 29813a22 | 127 | ___ret = READ_ONCE(*raw_cpu_ptr(&(pcp))); \ |
| 7f8d61f0 | 128 | preempt_enable_notrace(); \ |
| 29813a22 | 129 | ___ret; \ |
| 9c28278a TH |
130 | }) |
| 131 | ||
| e88d62cd MR |
132 | #define __this_cpu_generic_read_noirq(pcp) \ |
| 133 | ({ \ | |
| 29813a22 AB |
134 | typeof(pcp) ___ret; \ |
| 135 | unsigned long ___flags; \ | |
| 136 | raw_local_irq_save(___flags); \ | |
| 137 | ___ret = raw_cpu_generic_read(pcp); \ | |
| 138 | raw_local_irq_restore(___flags); \ | |
| 139 | ___ret; \ | |
| e88d62cd MR |
140 | }) |
| 141 | ||
| 142 | #define this_cpu_generic_read(pcp) \ | |
| 143 | ({ \ | |
| 144 | typeof(pcp) __ret; \ | |
| 145 | if (__native_word(pcp)) \ | |
| 146 | __ret = __this_cpu_generic_read_nopreempt(pcp); \ | |
| 147 | else \ | |
| 148 | __ret = __this_cpu_generic_read_noirq(pcp); \ | |
| 149 | __ret; \ | |
| 150 | }) | |
| 151 | ||
| eba11788 | 152 | #define this_cpu_generic_to_op(pcp, val, op) \ |
| 9c28278a | 153 | do { \ |
| eba11788 TH |
154 | unsigned long __flags; \ |
| 155 | raw_local_irq_save(__flags); \ | |
| 1b5ca121 | 156 | raw_cpu_generic_to_op(pcp, val, op); \ |
| eba11788 | 157 | raw_local_irq_restore(__flags); \ |
| 9c28278a TH |
158 | } while (0) |
| 159 | ||
| 1b5ca121 | 160 | |
| eba11788 | 161 | #define this_cpu_generic_add_return(pcp, val) \ |
| 9c28278a | 162 | ({ \ |
| eba11788 TH |
163 | typeof(pcp) __ret; \ |
| 164 | unsigned long __flags; \ | |
| 165 | raw_local_irq_save(__flags); \ | |
| 1b5ca121 | 166 | __ret = raw_cpu_generic_add_return(pcp, val); \ |
| eba11788 TH |
167 | raw_local_irq_restore(__flags); \ |
| 168 | __ret; \ | |
| 9c28278a TH |
169 | }) |
| 170 | ||
| eba11788 TH |
171 | #define this_cpu_generic_xchg(pcp, nval) \ |
| 172 | ({ \ | |
| 173 | typeof(pcp) __ret; \ | |
| 174 | unsigned long __flags; \ | |
| 175 | raw_local_irq_save(__flags); \ | |
| 1b5ca121 | 176 | __ret = raw_cpu_generic_xchg(pcp, nval); \ |
| eba11788 TH |
177 | raw_local_irq_restore(__flags); \ |
| 178 | __ret; \ | |
| 9c28278a TH |
179 | }) |
| 180 | ||
| c5c0ba95 PZ |
181 | #define this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \ |
| 182 | ({ \ | |
| 183 | bool __ret; \ | |
| 184 | unsigned long __flags; \ | |
| 185 | raw_local_irq_save(__flags); \ | |
| 186 | __ret = raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval); \ | |
| 187 | raw_local_irq_restore(__flags); \ | |
| 188 | __ret; \ | |
| 189 | }) | |
| 190 | ||
| eba11788 | 191 | #define this_cpu_generic_cmpxchg(pcp, oval, nval) \ |
| 9c28278a | 192 | ({ \ |
| eba11788 TH |
193 | typeof(pcp) __ret; \ |
| 194 | unsigned long __flags; \ | |
| 195 | raw_local_irq_save(__flags); \ | |
| 1b5ca121 | 196 | __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \ |
| eba11788 TH |
197 | raw_local_irq_restore(__flags); \ |
| 198 | __ret; \ | |
| 9c28278a TH |
199 | }) |
| 200 | ||
| eba11788 | 201 | #ifndef raw_cpu_read_1 |
| 1b5ca121 | 202 | #define raw_cpu_read_1(pcp) raw_cpu_generic_read(pcp) |
| eba11788 TH |
203 | #endif |
| 204 | #ifndef raw_cpu_read_2 | |
| 1b5ca121 | 205 | #define raw_cpu_read_2(pcp) raw_cpu_generic_read(pcp) |
| eba11788 TH |
206 | #endif |
| 207 | #ifndef raw_cpu_read_4 | |
| 1b5ca121 | 208 | #define raw_cpu_read_4(pcp) raw_cpu_generic_read(pcp) |
| eba11788 TH |
209 | #endif |
| 210 | #ifndef raw_cpu_read_8 | |
| 1b5ca121 | 211 | #define raw_cpu_read_8(pcp) raw_cpu_generic_read(pcp) |
| eba11788 TH |
212 | #endif |
| 213 | ||
| 214 | #ifndef raw_cpu_write_1 | |
| 215 | #define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op(pcp, val, =) | |
| 216 | #endif | |
| 217 | #ifndef raw_cpu_write_2 | |
| 218 | #define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op(pcp, val, =) | |
| 219 | #endif | |
| 220 | #ifndef raw_cpu_write_4 | |
| 221 | #define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op(pcp, val, =) | |
| 222 | #endif | |
| 223 | #ifndef raw_cpu_write_8 | |
| 224 | #define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op(pcp, val, =) | |
| 225 | #endif | |
| 226 | ||
| 227 | #ifndef raw_cpu_add_1 | |
| 228 | #define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op(pcp, val, +=) | |
| 229 | #endif | |
| 230 | #ifndef raw_cpu_add_2 | |
| 231 | #define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op(pcp, val, +=) | |
| 232 | #endif | |
| 233 | #ifndef raw_cpu_add_4 | |
| 234 | #define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op(pcp, val, +=) | |
| 235 | #endif | |
| 236 | #ifndef raw_cpu_add_8 | |
| 237 | #define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op(pcp, val, +=) | |
| 238 | #endif | |
| 239 | ||
| 240 | #ifndef raw_cpu_and_1 | |
| 241 | #define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op(pcp, val, &=) | |
| 242 | #endif | |
| 243 | #ifndef raw_cpu_and_2 | |
| 244 | #define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op(pcp, val, &=) | |
| 245 | #endif | |
| 246 | #ifndef raw_cpu_and_4 | |
| 247 | #define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op(pcp, val, &=) | |
| 248 | #endif | |
| 249 | #ifndef raw_cpu_and_8 | |
| 250 | #define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op(pcp, val, &=) | |
| 251 | #endif | |
| 252 | ||
| 253 | #ifndef raw_cpu_or_1 | |
| 254 | #define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op(pcp, val, |=) | |
| 255 | #endif | |
| 256 | #ifndef raw_cpu_or_2 | |
| 257 | #define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op(pcp, val, |=) | |
| 258 | #endif | |
| 259 | #ifndef raw_cpu_or_4 | |
| 260 | #define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op(pcp, val, |=) | |
| 261 | #endif | |
| 262 | #ifndef raw_cpu_or_8 | |
| 263 | #define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op(pcp, val, |=) | |
| 264 | #endif | |
| 265 | ||
| 266 | #ifndef raw_cpu_add_return_1 | |
| 267 | #define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val) | |
| 268 | #endif | |
| 269 | #ifndef raw_cpu_add_return_2 | |
| 270 | #define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val) | |
| 271 | #endif | |
| 272 | #ifndef raw_cpu_add_return_4 | |
| 273 | #define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val) | |
| 274 | #endif | |
| 275 | #ifndef raw_cpu_add_return_8 | |
| 276 | #define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val) | |
| 277 | #endif | |
| 278 | ||
| 279 | #ifndef raw_cpu_xchg_1 | |
| 280 | #define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval) | |
| 281 | #endif | |
| 282 | #ifndef raw_cpu_xchg_2 | |
| 283 | #define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval) | |
| 284 | #endif | |
| 285 | #ifndef raw_cpu_xchg_4 | |
| 286 | #define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval) | |
| 287 | #endif | |
| 288 | #ifndef raw_cpu_xchg_8 | |
| 289 | #define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval) | |
| 290 | #endif | |
| 291 | ||
| c5c0ba95 PZ |
292 | #ifndef raw_cpu_try_cmpxchg_1 |
| 293 | #ifdef raw_cpu_cmpxchg_1 | |
| 294 | #define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \ | |
| 295 | __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_1) | |
| 296 | #else | |
| 297 | #define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \ | |
| 298 | raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) | |
| 299 | #endif | |
| 300 | #endif | |
| 301 | #ifndef raw_cpu_try_cmpxchg_2 | |
| 302 | #ifdef raw_cpu_cmpxchg_2 | |
| 303 | #define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \ | |
| 304 | __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_2) | |
| 305 | #else | |
| 306 | #define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \ | |
| 307 | raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) | |
| 308 | #endif | |
| 309 | #endif | |
| 310 | #ifndef raw_cpu_try_cmpxchg_4 | |
| 311 | #ifdef raw_cpu_cmpxchg_4 | |
| 312 | #define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \ | |
| 313 | __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_4) | |
| 314 | #else | |
| 315 | #define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \ | |
| 316 | raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) | |
| 317 | #endif | |
| 318 | #endif | |
| 319 | #ifndef raw_cpu_try_cmpxchg_8 | |
| 320 | #ifdef raw_cpu_cmpxchg_8 | |
| 321 | #define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \ | |
| 322 | __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_8) | |
| 323 | #else | |
| 324 | #define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \ | |
| 325 | raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) | |
| 326 | #endif | |
| 327 | #endif | |
| 328 | ||
| 6d12c8d3 PZ |
329 | #ifndef raw_cpu_try_cmpxchg64 |
| 330 | #ifdef raw_cpu_cmpxchg64 | |
| 331 | #define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \ | |
| 332 | __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg64) | |
| 333 | #else | |
| 334 | #define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \ | |
| 335 | raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) | |
| 336 | #endif | |
| 337 | #endif | |
| 338 | #ifndef raw_cpu_try_cmpxchg128 | |
| 339 | #ifdef raw_cpu_cmpxchg128 | |
| 340 | #define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \ | |
| 341 | __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg128) | |
| 342 | #else | |
| 343 | #define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \ | |
| 344 | raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) | |
| 345 | #endif | |
| 346 | #endif | |
| 347 | ||
| eba11788 TH |
348 | #ifndef raw_cpu_cmpxchg_1 |
| 349 | #define raw_cpu_cmpxchg_1(pcp, oval, nval) \ | |
| 350 | raw_cpu_generic_cmpxchg(pcp, oval, nval) | |
| 351 | #endif | |
| 352 | #ifndef raw_cpu_cmpxchg_2 | |
| 353 | #define raw_cpu_cmpxchg_2(pcp, oval, nval) \ | |
| 354 | raw_cpu_generic_cmpxchg(pcp, oval, nval) | |
| 355 | #endif | |
| 356 | #ifndef raw_cpu_cmpxchg_4 | |
| 357 | #define raw_cpu_cmpxchg_4(pcp, oval, nval) \ | |
| 358 | raw_cpu_generic_cmpxchg(pcp, oval, nval) | |
| 359 | #endif | |
| 360 | #ifndef raw_cpu_cmpxchg_8 | |
| 361 | #define raw_cpu_cmpxchg_8(pcp, oval, nval) \ | |
| 362 | raw_cpu_generic_cmpxchg(pcp, oval, nval) | |
| 363 | #endif | |
| 364 | ||
| 6d12c8d3 PZ |
365 | #ifndef raw_cpu_cmpxchg64 |
| 366 | #define raw_cpu_cmpxchg64(pcp, oval, nval) \ | |
| 367 | raw_cpu_generic_cmpxchg(pcp, oval, nval) | |
| 368 | #endif | |
| 369 | #ifndef raw_cpu_cmpxchg128 | |
| 370 | #define raw_cpu_cmpxchg128(pcp, oval, nval) \ | |
| 371 | raw_cpu_generic_cmpxchg(pcp, oval, nval) | |
| 372 | #endif | |
| 373 | ||
| eba11788 TH |
374 | #ifndef this_cpu_read_1 |
| 375 | #define this_cpu_read_1(pcp) this_cpu_generic_read(pcp) | |
| 376 | #endif | |
| 377 | #ifndef this_cpu_read_2 | |
| 378 | #define this_cpu_read_2(pcp) this_cpu_generic_read(pcp) | |
| 379 | #endif | |
| 380 | #ifndef this_cpu_read_4 | |
| 381 | #define this_cpu_read_4(pcp) this_cpu_generic_read(pcp) | |
| 382 | #endif | |
| 383 | #ifndef this_cpu_read_8 | |
| 384 | #define this_cpu_read_8(pcp) this_cpu_generic_read(pcp) | |
| 385 | #endif | |
| 386 | ||
| 387 | #ifndef this_cpu_write_1 | |
| 388 | #define this_cpu_write_1(pcp, val) this_cpu_generic_to_op(pcp, val, =) | |
| 389 | #endif | |
| 390 | #ifndef this_cpu_write_2 | |
| 391 | #define this_cpu_write_2(pcp, val) this_cpu_generic_to_op(pcp, val, =) | |
| 392 | #endif | |
| 393 | #ifndef this_cpu_write_4 | |
| 394 | #define this_cpu_write_4(pcp, val) this_cpu_generic_to_op(pcp, val, =) | |
| 395 | #endif | |
| 396 | #ifndef this_cpu_write_8 | |
| 397 | #define this_cpu_write_8(pcp, val) this_cpu_generic_to_op(pcp, val, =) | |
| 398 | #endif | |
| 399 | ||
| 400 | #ifndef this_cpu_add_1 | |
| 401 | #define this_cpu_add_1(pcp, val) this_cpu_generic_to_op(pcp, val, +=) | |
| 402 | #endif | |
| 403 | #ifndef this_cpu_add_2 | |
| 404 | #define this_cpu_add_2(pcp, val) this_cpu_generic_to_op(pcp, val, +=) | |
| 405 | #endif | |
| 406 | #ifndef this_cpu_add_4 | |
| 407 | #define this_cpu_add_4(pcp, val) this_cpu_generic_to_op(pcp, val, +=) | |
| 408 | #endif | |
| 409 | #ifndef this_cpu_add_8 | |
| 410 | #define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=) | |
| 411 | #endif | |
| 412 | ||
| 413 | #ifndef this_cpu_and_1 | |
| 414 | #define this_cpu_and_1(pcp, val) this_cpu_generic_to_op(pcp, val, &=) | |
| 415 | #endif | |
| 416 | #ifndef this_cpu_and_2 | |
| 417 | #define this_cpu_and_2(pcp, val) this_cpu_generic_to_op(pcp, val, &=) | |
| 418 | #endif | |
| 419 | #ifndef this_cpu_and_4 | |
| 420 | #define this_cpu_and_4(pcp, val) this_cpu_generic_to_op(pcp, val, &=) | |
| 421 | #endif | |
| 422 | #ifndef this_cpu_and_8 | |
| 423 | #define this_cpu_and_8(pcp, val) this_cpu_generic_to_op(pcp, val, &=) | |
| 424 | #endif | |
| 425 | ||
| 426 | #ifndef this_cpu_or_1 | |
| 427 | #define this_cpu_or_1(pcp, val) this_cpu_generic_to_op(pcp, val, |=) | |
| 428 | #endif | |
| 429 | #ifndef this_cpu_or_2 | |
| 430 | #define this_cpu_or_2(pcp, val) this_cpu_generic_to_op(pcp, val, |=) | |
| 431 | #endif | |
| 432 | #ifndef this_cpu_or_4 | |
| 433 | #define this_cpu_or_4(pcp, val) this_cpu_generic_to_op(pcp, val, |=) | |
| 434 | #endif | |
| 435 | #ifndef this_cpu_or_8 | |
| 436 | #define this_cpu_or_8(pcp, val) this_cpu_generic_to_op(pcp, val, |=) | |
| 437 | #endif | |
| 438 | ||
| 439 | #ifndef this_cpu_add_return_1 | |
| 440 | #define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val) | |
| 441 | #endif | |
| 442 | #ifndef this_cpu_add_return_2 | |
| 443 | #define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val) | |
| 444 | #endif | |
| 445 | #ifndef this_cpu_add_return_4 | |
| 446 | #define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val) | |
| 447 | #endif | |
| 448 | #ifndef this_cpu_add_return_8 | |
| 449 | #define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val) | |
| 450 | #endif | |
| 451 | ||
| 452 | #ifndef this_cpu_xchg_1 | |
| 453 | #define this_cpu_xchg_1(pcp, nval) this_cpu_generic_xchg(pcp, nval) | |
| 454 | #endif | |
| 455 | #ifndef this_cpu_xchg_2 | |
| 456 | #define this_cpu_xchg_2(pcp, nval) this_cpu_generic_xchg(pcp, nval) | |
| 457 | #endif | |
| 458 | #ifndef this_cpu_xchg_4 | |
| 459 | #define this_cpu_xchg_4(pcp, nval) this_cpu_generic_xchg(pcp, nval) | |
| 460 | #endif | |
| 461 | #ifndef this_cpu_xchg_8 | |
| 462 | #define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval) | |
| 463 | #endif | |
| 464 | ||
| c5c0ba95 PZ |
465 | #ifndef this_cpu_try_cmpxchg_1 |
| 466 | #ifdef this_cpu_cmpxchg_1 | |
| 467 | #define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \ | |
| 468 | __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_1) | |
| 469 | #else | |
| 470 | #define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \ | |
| 471 | this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) | |
| 472 | #endif | |
| 473 | #endif | |
| 474 | #ifndef this_cpu_try_cmpxchg_2 | |
| 475 | #ifdef this_cpu_cmpxchg_2 | |
| 476 | #define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \ | |
| 477 | __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_2) | |
| 478 | #else | |
| 479 | #define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \ | |
| 480 | this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) | |
| 481 | #endif | |
| 482 | #endif | |
| 483 | #ifndef this_cpu_try_cmpxchg_4 | |
| 484 | #ifdef this_cpu_cmpxchg_4 | |
| 485 | #define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \ | |
| 486 | __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_4) | |
| 487 | #else | |
| 488 | #define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \ | |
| 489 | this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) | |
| 490 | #endif | |
| 491 | #endif | |
| 492 | #ifndef this_cpu_try_cmpxchg_8 | |
| 493 | #ifdef this_cpu_cmpxchg_8 | |
| 494 | #define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \ | |
| 495 | __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_8) | |
| 496 | #else | |
| 497 | #define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \ | |
| 498 | this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) | |
| 499 | #endif | |
| 500 | #endif | |
| 501 | ||
| 6d12c8d3 PZ |
502 | #ifndef this_cpu_try_cmpxchg64 |
| 503 | #ifdef this_cpu_cmpxchg64 | |
| 504 | #define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \ | |
| 505 | __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg64) | |
| 506 | #else | |
| 507 | #define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \ | |
| 508 | this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) | |
| 509 | #endif | |
| 510 | #endif | |
| 511 | #ifndef this_cpu_try_cmpxchg128 | |
| 512 | #ifdef this_cpu_cmpxchg128 | |
| 513 | #define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \ | |
| 514 | __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg128) | |
| 515 | #else | |
| 516 | #define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \ | |
| 517 | this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) | |
| 518 | #endif | |
| 519 | #endif | |
| 520 | ||
| eba11788 TH |
521 | #ifndef this_cpu_cmpxchg_1 |
| 522 | #define this_cpu_cmpxchg_1(pcp, oval, nval) \ | |
| 523 | this_cpu_generic_cmpxchg(pcp, oval, nval) | |
| 524 | #endif | |
| 525 | #ifndef this_cpu_cmpxchg_2 | |
| 526 | #define this_cpu_cmpxchg_2(pcp, oval, nval) \ | |
| 527 | this_cpu_generic_cmpxchg(pcp, oval, nval) | |
| 528 | #endif | |
| 529 | #ifndef this_cpu_cmpxchg_4 | |
| 530 | #define this_cpu_cmpxchg_4(pcp, oval, nval) \ | |
| 531 | this_cpu_generic_cmpxchg(pcp, oval, nval) | |
| 532 | #endif | |
| 533 | #ifndef this_cpu_cmpxchg_8 | |
| 534 | #define this_cpu_cmpxchg_8(pcp, oval, nval) \ | |
| 535 | this_cpu_generic_cmpxchg(pcp, oval, nval) | |
| 536 | #endif | |
| 537 | ||
| 6d12c8d3 PZ |
538 | #ifndef this_cpu_cmpxchg64 |
| 539 | #define this_cpu_cmpxchg64(pcp, oval, nval) \ | |
| 540 | this_cpu_generic_cmpxchg(pcp, oval, nval) | |
| 541 | #endif | |
| 542 | #ifndef this_cpu_cmpxchg128 | |
| 543 | #define this_cpu_cmpxchg128(pcp, oval, nval) \ | |
| 544 | this_cpu_generic_cmpxchg(pcp, oval, nval) | |
| 545 | #endif | |
| 546 | ||
| 1da177e4 | 547 | #endif /* _ASM_GENERIC_PERCPU_H_ */ |