Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
91d1aa43 FW |
2 | #ifndef _LINUX_CONTEXT_TRACKING_H |
3 | #define _LINUX_CONTEXT_TRACKING_H | |
4 | ||
91d1aa43 | 5 | #include <linux/sched.h> |
521921ba | 6 | #include <linux/vtime.h> |
e7358b3b | 7 | #include <linux/context_tracking_state.h> |
d19e789f IM |
8 | #include <linux/instrumentation.h> |
9 | ||
56dd9470 | 10 | #include <asm/ptrace.h> |
95a79fd4 | 11 | |
521921ba | 12 | |
24a9c541 | 13 | #ifdef CONFIG_CONTEXT_TRACKING_USER |
2a0aafce | 14 | extern void ct_cpu_track_user(int cpu); |
2e709338 | 15 | |
d0e536d8 | 16 | /* Called with interrupts disabled. */ |
0ffc781a FW |
17 | extern void __ct_user_enter(enum ctx_state state); |
18 | extern void __ct_user_exit(enum ctx_state state); | |
d0e536d8 | 19 | |
fe98db1c FW |
20 | extern void ct_user_enter(enum ctx_state state); |
21 | extern void ct_user_exit(enum ctx_state state); | |
22 | ||
f163f030 FW |
23 | extern void user_enter_callable(void); |
24 | extern void user_exit_callable(void); | |
ad65782f FW |
25 | |
26 | static inline void user_enter(void) | |
27 | { | |
74c57875 | 28 | if (context_tracking_enabled()) |
fe98db1c | 29 | ct_user_enter(CONTEXT_USER); |
ad65782f FW |
30 | |
31 | } | |
32 | static inline void user_exit(void) | |
33 | { | |
74c57875 | 34 | if (context_tracking_enabled()) |
fe98db1c | 35 | ct_user_exit(CONTEXT_USER); |
ad65782f | 36 | } |
56dd9470 | 37 | |
2e9d1e15 | 38 | /* Called with interrupts disabled. */ |
0372007f | 39 | static __always_inline void user_enter_irqoff(void) |
2e9d1e15 | 40 | { |
74c57875 | 41 | if (context_tracking_enabled()) |
0ffc781a | 42 | __ct_user_enter(CONTEXT_USER); |
2e9d1e15 PB |
43 | |
44 | } | |
0372007f | 45 | static __always_inline void user_exit_irqoff(void) |
2e9d1e15 | 46 | { |
74c57875 | 47 | if (context_tracking_enabled()) |
0ffc781a | 48 | __ct_user_exit(CONTEXT_USER); |
2e9d1e15 PB |
49 | } |
50 | ||
6c1e0256 | 51 | static inline enum ctx_state exception_enter(void) |
56dd9470 | 52 | { |
6c1e0256 FW |
53 | enum ctx_state prev_ctx; |
54 | ||
24a9c541 | 55 | if (IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) || |
179a9cf7 | 56 | !context_tracking_enabled()) |
ad65782f FW |
57 | return 0; |
58 | ||
17147677 | 59 | prev_ctx = __ct_state(); |
3aab4f50 | 60 | if (prev_ctx != CONTEXT_KERNEL) |
fe98db1c | 61 | ct_user_exit(prev_ctx); |
6c1e0256 FW |
62 | |
63 | return prev_ctx; | |
56dd9470 FW |
64 | } |
65 | ||
6c1e0256 | 66 | static inline void exception_exit(enum ctx_state prev_ctx) |
56dd9470 | 67 | { |
24a9c541 | 68 | if (!IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) && |
179a9cf7 | 69 | context_tracking_enabled()) { |
3aab4f50 | 70 | if (prev_ctx != CONTEXT_KERNEL) |
fe98db1c | 71 | ct_user_enter(prev_ctx); |
ad65782f | 72 | } |
56dd9470 FW |
73 | } |
74 | ||
14296e0c SC |
75 | static __always_inline bool context_tracking_guest_enter(void) |
76 | { | |
77 | if (context_tracking_enabled()) | |
0ffc781a | 78 | __ct_user_enter(CONTEXT_GUEST); |
14296e0c SC |
79 | |
80 | return context_tracking_enabled_this_cpu(); | |
81 | } | |
82 | ||
83 | static __always_inline void context_tracking_guest_exit(void) | |
84 | { | |
85 | if (context_tracking_enabled()) | |
0ffc781a | 86 | __ct_user_exit(CONTEXT_GUEST); |
14296e0c | 87 | } |
f9281648 | 88 | |
17147677 FW |
89 | #define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond)) |
90 | ||
91d1aa43 FW |
91 | #else |
92 | static inline void user_enter(void) { } | |
93 | static inline void user_exit(void) { } | |
2e9d1e15 PB |
94 | static inline void user_enter_irqoff(void) { } |
95 | static inline void user_exit_irqoff(void) { } | |
17147677 | 96 | static inline int exception_enter(void) { return 0; } |
2d854e57 | 97 | static inline void exception_exit(enum ctx_state prev_ctx) { } |
17147677 | 98 | static inline int ct_state(void) { return -1; } |
f87d2867 | 99 | static inline int __ct_state(void) { return -1; } |
e25b694b | 100 | static __always_inline bool context_tracking_guest_enter(void) { return false; } |
e8deb00c | 101 | static __always_inline void context_tracking_guest_exit(void) { } |
17147677 | 102 | #define CT_WARN_ON(cond) do { } while (0) |
24a9c541 | 103 | #endif /* !CONFIG_CONTEXT_TRACKING_USER */ |
521921ba | 104 | |
24a9c541 | 105 | #ifdef CONFIG_CONTEXT_TRACKING_USER_FORCE |
65f382fd FW |
106 | extern void context_tracking_init(void); |
107 | #else | |
108 | static inline void context_tracking_init(void) { } | |
24a9c541 | 109 | #endif /* CONFIG_CONTEXT_TRACKING_USER_FORCE */ |
65f382fd | 110 | |
e67198cc FW |
111 | #ifdef CONFIG_CONTEXT_TRACKING_IDLE |
112 | extern void ct_idle_enter(void); | |
113 | extern void ct_idle_exit(void); | |
17211455 FW |
114 | |
115 | /* | |
116 | * Is the current CPU in an extended quiescent state? | |
117 | * | |
118 | * No ordering, as we are sampling CPU-local information. | |
119 | */ | |
120 | static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) | |
121 | { | |
0f613bfa | 122 | return !(raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & RCU_DYNTICKS_IDX); |
17211455 FW |
123 | } |
124 | ||
125 | /* | |
17147677 | 126 | * Increment the current CPU's context_tracking structure's ->state field |
17211455 FW |
127 | * with ordering. Return the new value. |
128 | */ | |
17147677 | 129 | static __always_inline unsigned long ct_state_inc(int incby) |
17211455 | 130 | { |
0f613bfa | 131 | return raw_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state)); |
17211455 FW |
132 | } |
133 | ||
5a5d7e9b PZ |
134 | static __always_inline bool warn_rcu_enter(void) |
135 | { | |
136 | bool ret = false; | |
137 | ||
138 | /* | |
139 | * Horrible hack to shut up recursive RCU isn't watching fail since | |
140 | * lots of the actual reporting also relies on RCU. | |
141 | */ | |
142 | preempt_disable_notrace(); | |
143 | if (rcu_dynticks_curr_cpu_in_eqs()) { | |
144 | ret = true; | |
145 | ct_state_inc(RCU_DYNTICKS_IDX); | |
146 | } | |
147 | ||
148 | return ret; | |
149 | } | |
150 | ||
151 | static __always_inline void warn_rcu_exit(bool rcu) | |
152 | { | |
153 | if (rcu) | |
154 | ct_state_inc(RCU_DYNTICKS_IDX); | |
155 | preempt_enable_notrace(); | |
156 | } | |
157 | ||
e67198cc FW |
158 | #else |
159 | static inline void ct_idle_enter(void) { } | |
160 | static inline void ct_idle_exit(void) { } | |
5a5d7e9b PZ |
161 | |
162 | static __always_inline bool warn_rcu_enter(void) { return false; } | |
163 | static __always_inline void warn_rcu_exit(bool rcu) { } | |
e67198cc FW |
164 | #endif /* !CONFIG_CONTEXT_TRACKING_IDLE */ |
165 | ||
91d1aa43 | 166 | #endif |