Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
6abcd98f GOC |
2 | #ifndef _X86_IRQFLAGS_H_ |
3 | #define _X86_IRQFLAGS_H_ | |
4 | ||
5 | #include <asm/processor-flags.h> | |
6 | ||
7 | #ifndef __ASSEMBLY__ | |
6727ad9e CM |
8 | |
9 | /* Provide __cpuidle; we can't safely include <linux/cpu.h> */ | |
10 | #define __cpuidle __attribute__((__section__(".cpuidle.text"))) | |
11 | ||
6abcd98f GOC |
12 | /* |
13 | * Interrupt control: | |
14 | */ | |
15 | ||
208cbb32 ND |
16 | /* Declaration required for gcc < 4.9 to prevent -Werror=missing-prototypes */ |
17 | extern inline unsigned long native_save_fl(void); | |
d0a8d937 | 18 | extern inline unsigned long native_save_fl(void) |
6abcd98f GOC |
19 | { |
20 | unsigned long flags; | |
21 | ||
f1f029c7 | 22 | /* |
ab94fcf5 PA |
23 | * "=rm" is safe here, because "pop" adjusts the stack before |
24 | * it evaluates its effective address -- this is part of the | |
25 | * documented behavior of the "pop" instruction. | |
f1f029c7 | 26 | */ |
cf7f7191 JP |
27 | asm volatile("# __raw_save_flags\n\t" |
28 | "pushf ; pop %0" | |
ab94fcf5 | 29 | : "=rm" (flags) |
cf7f7191 JP |
30 | : /* no input */ |
31 | : "memory"); | |
6abcd98f GOC |
32 | |
33 | return flags; | |
34 | } | |
35 | ||
1f59a458 ND |
36 | extern inline void native_restore_fl(unsigned long flags); |
37 | extern inline void native_restore_fl(unsigned long flags) | |
6abcd98f | 38 | { |
cf7f7191 JP |
39 | asm volatile("push %0 ; popf" |
40 | : /* no output */ | |
41 | :"g" (flags) | |
42 | :"memory", "cc"); | |
6abcd98f GOC |
43 | } |
44 | ||
45 | static inline void native_irq_disable(void) | |
46 | { | |
47 | asm volatile("cli": : :"memory"); | |
48 | } | |
49 | ||
50 | static inline void native_irq_enable(void) | |
51 | { | |
52 | asm volatile("sti": : :"memory"); | |
53 | } | |
54 | ||
6727ad9e | 55 | static inline __cpuidle void native_safe_halt(void) |
6abcd98f GOC |
56 | { |
57 | asm volatile("sti; hlt": : :"memory"); | |
58 | } | |
59 | ||
6727ad9e | 60 | static inline __cpuidle void native_halt(void) |
6abcd98f GOC |
61 | { |
62 | asm volatile("hlt": : :"memory"); | |
63 | } | |
64 | ||
65 | #endif | |
66 | ||
67 | #ifdef CONFIG_PARAVIRT | |
68 | #include <asm/paravirt.h> | |
69 | #else | |
70 | #ifndef __ASSEMBLY__ | |
e08fbb78 | 71 | #include <linux/types.h> |
6abcd98f | 72 | |
e08fbb78 | 73 | static inline notrace unsigned long arch_local_save_flags(void) |
6abcd98f GOC |
74 | { |
75 | return native_save_fl(); | |
76 | } | |
77 | ||
e08fbb78 | 78 | static inline notrace void arch_local_irq_restore(unsigned long flags) |
6abcd98f GOC |
79 | { |
80 | native_restore_fl(flags); | |
81 | } | |
82 | ||
e08fbb78 | 83 | static inline notrace void arch_local_irq_disable(void) |
6abcd98f GOC |
84 | { |
85 | native_irq_disable(); | |
86 | } | |
87 | ||
e08fbb78 | 88 | static inline notrace void arch_local_irq_enable(void) |
6abcd98f GOC |
89 | { |
90 | native_irq_enable(); | |
91 | } | |
92 | ||
93 | /* | |
94 | * Used in the idle loop; sti takes one instruction cycle | |
95 | * to complete: | |
96 | */ | |
6727ad9e | 97 | static inline __cpuidle void arch_safe_halt(void) |
6abcd98f GOC |
98 | { |
99 | native_safe_halt(); | |
100 | } | |
101 | ||
102 | /* | |
103 | * Used when interrupts are already enabled or to | |
104 | * shutdown the processor: | |
105 | */ | |
6727ad9e | 106 | static inline __cpuidle void halt(void) |
6abcd98f GOC |
107 | { |
108 | native_halt(); | |
109 | } | |
110 | ||
111 | /* | |
112 | * For spinlocks, etc: | |
113 | */ | |
e08fbb78 | 114 | static inline notrace unsigned long arch_local_irq_save(void) |
6abcd98f | 115 | { |
df9ee292 DH |
116 | unsigned long flags = arch_local_save_flags(); |
117 | arch_local_irq_disable(); | |
6abcd98f GOC |
118 | return flags; |
119 | } | |
120 | #else | |
121 | ||
122 | #define ENABLE_INTERRUPTS(x) sti | |
123 | #define DISABLE_INTERRUPTS(x) cli | |
124 | ||
125 | #ifdef CONFIG_X86_64 | |
df366e98 JF |
126 | #define SWAPGS swapgs |
127 | /* | |
128 | * Currently paravirt can't handle swapgs nicely when we | |
129 | * don't have a stack we can rely on (such as a user space | |
130 | * stack). So we either find a way around these or just fault | |
131 | * and emulate if a guest tries to call swapgs directly. | |
132 | * | |
133 | * Either way, this is a good way to document that we don't | |
134 | * have a reliable stack. x86_64 only. | |
135 | */ | |
a00394f8 | 136 | #define SWAPGS_UNSAFE_STACK swapgs |
df366e98 JF |
137 | |
138 | #define PARAVIRT_ADJUST_EXCEPTION_FRAME /* */ | |
139 | ||
7209a75d | 140 | #define INTERRUPT_RETURN jmp native_iret |
2be29982 JF |
141 | #define USERGS_SYSRET64 \ |
142 | swapgs; \ | |
143 | sysretq; | |
144 | #define USERGS_SYSRET32 \ | |
145 | swapgs; \ | |
146 | sysretl | |
2be29982 | 147 | |
e17f8234 BO |
148 | #ifdef CONFIG_DEBUG_ENTRY |
149 | #define SAVE_FLAGS(x) pushfq; popq %rax | |
150 | #endif | |
6abcd98f GOC |
151 | #else |
152 | #define INTERRUPT_RETURN iret | |
d75cd22f | 153 | #define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit |
6abcd98f GOC |
154 | #define GET_CR0_INTO_EAX movl %cr0, %eax |
155 | #endif | |
156 | ||
157 | ||
158 | #endif /* __ASSEMBLY__ */ | |
159 | #endif /* CONFIG_PARAVIRT */ | |
160 | ||
161 | #ifndef __ASSEMBLY__ | |
df9ee292 | 162 | static inline int arch_irqs_disabled_flags(unsigned long flags) |
6abcd98f GOC |
163 | { |
164 | return !(flags & X86_EFLAGS_IF); | |
165 | } | |
166 | ||
df9ee292 | 167 | static inline int arch_irqs_disabled(void) |
6abcd98f | 168 | { |
df9ee292 | 169 | unsigned long flags = arch_local_save_flags(); |
6abcd98f | 170 | |
df9ee292 | 171 | return arch_irqs_disabled_flags(flags); |
6abcd98f | 172 | } |
40e2ec65 | 173 | #endif /* !__ASSEMBLY__ */ |
6abcd98f | 174 | |
40e2ec65 DV |
175 | #ifdef __ASSEMBLY__ |
176 | #ifdef CONFIG_TRACE_IRQFLAGS | |
177 | # define TRACE_IRQS_ON call trace_hardirqs_on_thunk; | |
178 | # define TRACE_IRQS_OFF call trace_hardirqs_off_thunk; | |
96a388de | 179 | #else |
40e2ec65 DV |
180 | # define TRACE_IRQS_ON |
181 | # define TRACE_IRQS_OFF | |
182 | #endif | |
183 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | |
184 | # ifdef CONFIG_X86_64 | |
7dc7cc07 DV |
185 | # define LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk |
186 | # define LOCKDEP_SYS_EXIT_IRQ \ | |
6abcd98f GOC |
187 | TRACE_IRQS_ON; \ |
188 | sti; \ | |
7dc7cc07 | 189 | call lockdep_sys_exit_thunk; \ |
6abcd98f GOC |
190 | cli; \ |
191 | TRACE_IRQS_OFF; | |
40e2ec65 | 192 | # else |
7dc7cc07 | 193 | # define LOCKDEP_SYS_EXIT \ |
6abcd98f GOC |
194 | pushl %eax; \ |
195 | pushl %ecx; \ | |
196 | pushl %edx; \ | |
197 | call lockdep_sys_exit; \ | |
198 | popl %edx; \ | |
199 | popl %ecx; \ | |
200 | popl %eax; | |
7dc7cc07 | 201 | # define LOCKDEP_SYS_EXIT_IRQ |
40e2ec65 | 202 | # endif |
7dc7cc07 | 203 | #else |
6abcd98f GOC |
204 | # define LOCKDEP_SYS_EXIT |
205 | # define LOCKDEP_SYS_EXIT_IRQ | |
7dc7cc07 | 206 | #endif |
6abcd98f | 207 | #endif /* __ASSEMBLY__ */ |
40e2ec65 | 208 | |
96a388de | 209 | #endif |