Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
91d1aa43 FW |
2 | #ifndef _LINUX_CONTEXT_TRACKING_H |
3 | #define _LINUX_CONTEXT_TRACKING_H | |
4 | ||
91d1aa43 | 5 | #include <linux/sched.h> |
521921ba | 6 | #include <linux/vtime.h> |
e7358b3b | 7 | #include <linux/context_tracking_state.h> |
56dd9470 | 8 | #include <asm/ptrace.h> |
95a79fd4 | 9 | |
521921ba | 10 | |
6c1e0256 | 11 | #ifdef CONFIG_CONTEXT_TRACKING |
2e709338 FW |
12 | extern void context_tracking_cpu_set(int cpu); |
13 | ||
d0e536d8 PB |
14 | /* Called with interrupts disabled. */ |
15 | extern void __context_tracking_enter(enum ctx_state state); | |
16 | extern void __context_tracking_exit(enum ctx_state state); | |
17 | ||
3aab4f50 RR |
18 | extern void context_tracking_enter(enum ctx_state state); |
19 | extern void context_tracking_exit(enum ctx_state state); | |
ad65782f FW |
20 | extern void context_tracking_user_enter(void); |
21 | extern void context_tracking_user_exit(void); | |
22 | ||
23 | static inline void user_enter(void) | |
24 | { | |
58135f57 | 25 | if (context_tracking_is_enabled()) |
f70cd6b0 | 26 | context_tracking_enter(CONTEXT_USER); |
ad65782f FW |
27 | |
28 | } | |
29 | static inline void user_exit(void) | |
30 | { | |
58135f57 | 31 | if (context_tracking_is_enabled()) |
f70cd6b0 | 32 | context_tracking_exit(CONTEXT_USER); |
ad65782f | 33 | } |
56dd9470 | 34 | |
2e9d1e15 PB |
35 | /* Called with interrupts disabled. */ |
36 | static inline void user_enter_irqoff(void) | |
37 | { | |
38 | if (context_tracking_is_enabled()) | |
39 | __context_tracking_enter(CONTEXT_USER); | |
40 | ||
41 | } | |
42 | static inline void user_exit_irqoff(void) | |
43 | { | |
44 | if (context_tracking_is_enabled()) | |
45 | __context_tracking_exit(CONTEXT_USER); | |
46 | } | |
47 | ||
6c1e0256 | 48 | static inline enum ctx_state exception_enter(void) |
56dd9470 | 49 | { |
6c1e0256 FW |
50 | enum ctx_state prev_ctx; |
51 | ||
58135f57 | 52 | if (!context_tracking_is_enabled()) |
ad65782f FW |
53 | return 0; |
54 | ||
6c1e0256 | 55 | prev_ctx = this_cpu_read(context_tracking.state); |
3aab4f50 RR |
56 | if (prev_ctx != CONTEXT_KERNEL) |
57 | context_tracking_exit(prev_ctx); | |
6c1e0256 FW |
58 | |
59 | return prev_ctx; | |
56dd9470 FW |
60 | } |
61 | ||
6c1e0256 | 62 | static inline void exception_exit(enum ctx_state prev_ctx) |
56dd9470 | 63 | { |
58135f57 | 64 | if (context_tracking_is_enabled()) { |
3aab4f50 RR |
65 | if (prev_ctx != CONTEXT_KERNEL) |
66 | context_tracking_enter(prev_ctx); | |
ad65782f | 67 | } |
56dd9470 FW |
68 | } |
69 | ||
f9281648 AL |
70 | |
71 | /** | |
72 | * ct_state() - return the current context tracking state if known | |
73 | * | |
74 | * Returns the current cpu's context tracking state if context tracking | |
75 | * is enabled. If context tracking is disabled, returns | |
76 | * CONTEXT_DISABLED. This should be used primarily for debugging. | |
77 | */ | |
78 | static inline enum ctx_state ct_state(void) | |
79 | { | |
80 | return context_tracking_is_enabled() ? | |
81 | this_cpu_read(context_tracking.state) : CONTEXT_DISABLED; | |
82 | } | |
91d1aa43 FW |
83 | #else |
84 | static inline void user_enter(void) { } | |
85 | static inline void user_exit(void) { } | |
2e9d1e15 PB |
86 | static inline void user_enter_irqoff(void) { } |
87 | static inline void user_exit_irqoff(void) { } | |
2d854e57 FW |
88 | static inline enum ctx_state exception_enter(void) { return 0; } |
89 | static inline void exception_exit(enum ctx_state prev_ctx) { } | |
f9281648 | 90 | static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; } |
2d854e57 | 91 | #endif /* !CONFIG_CONTEXT_TRACKING */ |
521921ba | 92 | |
f9281648 | 93 | #define CT_WARN_ON(cond) WARN_ON(context_tracking_is_enabled() && (cond)) |
65f382fd FW |
94 | |
95 | #ifdef CONFIG_CONTEXT_TRACKING_FORCE | |
96 | extern void context_tracking_init(void); | |
97 | #else | |
98 | static inline void context_tracking_init(void) { } | |
99 | #endif /* CONFIG_CONTEXT_TRACKING_FORCE */ | |
100 | ||
101 | ||
2d854e57 | 102 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN |
ebaac173 PB |
103 | /* must be called with irqs disabled */ |
104 | static inline void guest_enter_irqoff(void) | |
48d6a816 | 105 | { |
55dbdcfa | 106 | if (vtime_accounting_cpu_enabled()) |
48d6a816 FW |
107 | vtime_guest_enter(current); |
108 | else | |
109 | current->flags |= PF_VCPU; | |
126a6a54 RR |
110 | |
111 | if (context_tracking_is_enabled()) | |
d0e536d8 | 112 | __context_tracking_enter(CONTEXT_GUEST); |
ebaac173 PB |
113 | |
114 | /* KVM does not hold any references to rcu protected data when it | |
115 | * switches CPU into a guest mode. In fact switching to a guest mode | |
116 | * is very similar to exiting to userspace from rcu point of view. In | |
117 | * addition CPU may stay in a guest mode for quite a long time (up to | |
118 | * one time slice). Lets treat guest mode as quiescent state, just like | |
119 | * we do with user-mode execution. | |
120 | */ | |
121 | if (!context_tracking_cpu_is_enabled()) | |
122 | rcu_virt_note_context_switch(smp_processor_id()); | |
48d6a816 FW |
123 | } |
124 | ||
ebaac173 | 125 | static inline void guest_exit_irqoff(void) |
48d6a816 | 126 | { |
126a6a54 | 127 | if (context_tracking_is_enabled()) |
d0e536d8 | 128 | __context_tracking_exit(CONTEXT_GUEST); |
126a6a54 | 129 | |
55dbdcfa | 130 | if (vtime_accounting_cpu_enabled()) |
48d6a816 FW |
131 | vtime_guest_exit(current); |
132 | else | |
133 | current->flags &= ~PF_VCPU; | |
134 | } | |
73d424f9 | 135 | |
2d854e57 | 136 | #else |
ebaac173 | 137 | static inline void guest_enter_irqoff(void) |
521921ba | 138 | { |
2d854e57 | 139 | /* |
5b206d48 FW |
140 | * This is running in ioctl context so its safe |
141 | * to assume that it's the stime pending cputime | |
142 | * to flush. | |
2d854e57 FW |
143 | */ |
144 | vtime_account_system(current); | |
145 | current->flags |= PF_VCPU; | |
ebaac173 | 146 | rcu_virt_note_context_switch(smp_processor_id()); |
521921ba FW |
147 | } |
148 | ||
ebaac173 | 149 | static inline void guest_exit_irqoff(void) |
521921ba | 150 | { |
5b206d48 | 151 | /* Flush the guest cputime we spent on the guest */ |
2d854e57 FW |
152 | vtime_account_system(current); |
153 | current->flags &= ~PF_VCPU; | |
521921ba | 154 | } |
2d854e57 | 155 | #endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */ |
91d1aa43 | 156 | |
ebaac173 PB |
157 | static inline void guest_enter(void) |
158 | { | |
159 | unsigned long flags; | |
160 | ||
161 | local_irq_save(flags); | |
162 | guest_enter_irqoff(); | |
163 | local_irq_restore(flags); | |
164 | } | |
165 | ||
166 | static inline void guest_exit(void) | |
167 | { | |
168 | unsigned long flags; | |
169 | ||
170 | local_irq_save(flags); | |
171 | guest_exit_irqoff(); | |
172 | local_irq_restore(flags); | |
173 | } | |
174 | ||
91d1aa43 | 175 | #endif |