Commit | Line | Data |
---|---|---|
e506ea45 WD |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* | |
3 | * Prevent the compiler from merging or refetching reads or writes. The | |
4 | * compiler is also forbidden from reordering successive instances of | |
5 | * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some | |
6 | * particular ordering. One way to make the compiler aware of ordering is to | |
7 | * put the two invocations of READ_ONCE or WRITE_ONCE in different C | |
8 | * statements. | |
9 | * | |
10 | * These two macros will also work on aggregate data types like structs or | |
11 | * unions. | |
12 | * | |
13 | * Their two major use cases are: (1) Mediating communication between | |
14 | * process-level code and irq/NMI handlers, all running on the same CPU, | |
15 | * and (2) Ensuring that the compiler does not fold, spindle, or otherwise | |
16 | * mutilate accesses that either do not require ordering or that interact | |
17 | * with an explicit memory barrier or atomic instruction that provides the | |
18 | * required ordering. | |
19 | */ | |
20 | #ifndef __ASM_GENERIC_RWONCE_H | |
21 | #define __ASM_GENERIC_RWONCE_H | |
22 | ||
23 | #ifndef __ASSEMBLY__ | |
24 | ||
25 | #include <linux/compiler_types.h> | |
26 | #include <linux/kasan-checks.h> | |
27 | #include <linux/kcsan-checks.h> | |
28 | ||
e506ea45 WD |
29 | /* |
30 | * Yes, this permits 64-bit accesses on 32-bit architectures. These will | |
31 | * actually be atomic in some cases (namely Armv7 + LPAE), but for others we | |
32 | * rely on the access being split into 2x32-bit accesses for a 32-bit quantity | |
33 | * (e.g. a virtual address) and a strong prevailing wind. | |
34 | */ | |
35 | #define compiletime_assert_rwonce_type(t) \ | |
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \ | |
37 | "Unsupported access size for {READ,WRITE}_ONCE().") | |
38 | ||
39 | /* | |
40 | * Use __READ_ONCE() instead of READ_ONCE() if you do not require any | |
3c918410 | 41 | * atomicity. Note that this may result in tears! |
e506ea45 | 42 | */ |
b78b331a | 43 | #ifndef __READ_ONCE |
e506ea45 | 44 | #define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x)) |
b78b331a | 45 | #endif |
e506ea45 | 46 | |
e506ea45 WD |
47 | #define READ_ONCE(x) \ |
48 | ({ \ | |
49 | compiletime_assert_rwonce_type(x); \ | |
3c918410 | 50 | __READ_ONCE(x); \ |
e506ea45 WD |
51 | }) |
52 | ||
53 | #define __WRITE_ONCE(x, val) \ | |
54 | do { \ | |
55 | *(volatile typeof(x) *)&(x) = (val); \ | |
56 | } while (0) | |
57 | ||
58 | #define WRITE_ONCE(x, val) \ | |
59 | do { \ | |
60 | compiletime_assert_rwonce_type(x); \ | |
61 | __WRITE_ONCE(x, val); \ | |
62 | } while (0) | |
63 | ||
64 | static __no_sanitize_or_inline | |
65 | unsigned long __read_once_word_nocheck(const void *addr) | |
66 | { | |
67 | return __READ_ONCE(*(unsigned long *)addr); | |
68 | } | |
69 | ||
70 | /* | |
71 | * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need to load a | |
72 | * word from memory atomically but without telling KASAN/KCSAN. This is | |
73 | * usually used by unwinding code when walking the stack of a running process. | |
74 | */ | |
75 | #define READ_ONCE_NOCHECK(x) \ | |
76 | ({ \ | |
3c918410 | 77 | compiletime_assert(sizeof(x) == sizeof(unsigned long), \ |
e506ea45 | 78 | "Unsupported access size for READ_ONCE_NOCHECK()."); \ |
3c918410 | 79 | (typeof(x))__read_once_word_nocheck(&(x)); \ |
e506ea45 WD |
80 | }) |
81 | ||
82 | static __no_kasan_or_inline | |
83 | unsigned long read_word_at_a_time(const void *addr) | |
84 | { | |
85 | kasan_check_read(addr, 1); | |
86 | return *(unsigned long *)addr; | |
87 | } | |
88 | ||
89 | #endif /* __ASSEMBLY__ */ | |
90 | #endif /* __ASM_GENERIC_RWONCE_H */ |