Commit | Line | Data |
---|---|---|
744c193e | 1 | #include <linux/extable.h> |
7c0f6ba6 | 2 | #include <linux/uaccess.h> |
b17b0153 | 3 | #include <linux/sched/debug.h> |
42b3a4cb | 4 | #include <xen/xen.h> |
b17b0153 | 5 | |
d5c8028b | 6 | #include <asm/fpu/internal.h> |
0d0efc07 | 7 | #include <asm/traps.h> |
81c2949f | 8 | #include <asm/kdebug.h> |
6d48583b | 9 | |
548acf19 | 10 | typedef bool (*ex_handler_t)(const struct exception_table_entry *, |
81fd9c18 JH |
11 | struct pt_regs *, int, unsigned long, |
12 | unsigned long); | |
548acf19 | 13 | |
70627654 PA |
14 | static inline unsigned long |
15 | ex_fixup_addr(const struct exception_table_entry *x) | |
16 | { | |
17 | return (unsigned long)&x->fixup + x->fixup; | |
18 | } | |
548acf19 TL |
19 | static inline ex_handler_t |
20 | ex_fixup_handler(const struct exception_table_entry *x) | |
21 | { | |
22 | return (ex_handler_t)((unsigned long)&x->handler + x->handler); | |
23 | } | |
6d48583b | 24 | |
80a3e394 | 25 | __visible bool ex_handler_default(const struct exception_table_entry *fixup, |
81fd9c18 JH |
26 | struct pt_regs *regs, int trapnr, |
27 | unsigned long error_code, | |
28 | unsigned long fault_addr) | |
6d48583b | 29 | { |
548acf19 TL |
30 | regs->ip = ex_fixup_addr(fixup); |
31 | return true; | |
32 | } | |
33 | EXPORT_SYMBOL(ex_handler_default); | |
34 | ||
80a3e394 | 35 | __visible bool ex_handler_fault(const struct exception_table_entry *fixup, |
81fd9c18 JH |
36 | struct pt_regs *regs, int trapnr, |
37 | unsigned long error_code, | |
38 | unsigned long fault_addr) | |
548acf19 TL |
39 | { |
40 | regs->ip = ex_fixup_addr(fixup); | |
41 | regs->ax = trapnr; | |
42 | return true; | |
43 | } | |
44 | EXPORT_SYMBOL_GPL(ex_handler_fault); | |
45 | ||
7a46ec0e KC |
46 | /* |
47 | * Handler for UD0 exception following a failed test against the | |
48 | * result of a refcount inc/dec/add/sub. | |
49 | */ | |
80a3e394 | 50 | __visible bool ex_handler_refcount(const struct exception_table_entry *fixup, |
81fd9c18 JH |
51 | struct pt_regs *regs, int trapnr, |
52 | unsigned long error_code, | |
53 | unsigned long fault_addr) | |
7a46ec0e KC |
54 | { |
55 | /* First unconditionally saturate the refcount. */ | |
56 | *(int *)regs->cx = INT_MIN / 2; | |
57 | ||
58 | /* | |
59 | * Strictly speaking, this reports the fixup destination, not | |
60 | * the fault location, and not the actually overflowing | |
61 | * instruction, which is the instruction before the "js", but | |
62 | * since that instruction could be a variety of lengths, just | |
63 | * report the location after the overflow, which should be close | |
64 | * enough for finding the overflow, as it's at least back in | |
65 | * the function, having returned from .text.unlikely. | |
66 | */ | |
67 | regs->ip = ex_fixup_addr(fixup); | |
68 | ||
69 | /* | |
70 | * This function has been called because either a negative refcount | |
71 | * value was seen by any of the refcount functions, or a zero | |
72 | * refcount value was seen by refcount_dec(). | |
73 | * | |
74 | * If we crossed from INT_MAX to INT_MIN, OF (Overflow Flag: result | |
75 | * wrapped around) will be set. Additionally, seeing the refcount | |
76 | * reach 0 will set ZF (Zero Flag: result was zero). In each of | |
77 | * these cases we want a report, since it's a boundary condition. | |
564c9cc8 KC |
78 | * The SF case is not reported since it indicates post-boundary |
79 | * manipulations below zero or above INT_MAX. And if none of the | |
80 | * flags are set, something has gone very wrong, so report it. | |
7a46ec0e KC |
81 | */ |
82 | if (regs->flags & (X86_EFLAGS_OF | X86_EFLAGS_ZF)) { | |
83 | bool zero = regs->flags & X86_EFLAGS_ZF; | |
84 | ||
85 | refcount_error_report(regs, zero ? "hit zero" : "overflow"); | |
564c9cc8 KC |
86 | } else if ((regs->flags & X86_EFLAGS_SF) == 0) { |
87 | /* Report if none of OF, ZF, nor SF are set. */ | |
88 | refcount_error_report(regs, "unexpected saturation"); | |
7a46ec0e KC |
89 | } |
90 | ||
91 | return true; | |
92 | } | |
b562c171 | 93 | EXPORT_SYMBOL(ex_handler_refcount); |
7a46ec0e | 94 | |
d5c8028b EB |
95 | /* |
96 | * Handler for when we fail to restore a task's FPU state. We should never get | |
97 | * here because the FPU state of a task using the FPU (task->thread.fpu.state) | |
98 | * should always be valid. However, past bugs have allowed userspace to set | |
99 | * reserved bits in the XSAVE area using PTRACE_SETREGSET or sys_rt_sigreturn(). | |
100 | * These caused XRSTOR to fail when switching to the task, leaking the FPU | |
101 | * registers of the task previously executing on the CPU. Mitigate this class | |
102 | * of vulnerability by restoring from the initial state (essentially, zeroing | |
103 | * out all the FPU registers) if we can't restore from the task's FPU state. | |
104 | */ | |
80a3e394 | 105 | __visible bool ex_handler_fprestore(const struct exception_table_entry *fixup, |
81fd9c18 JH |
106 | struct pt_regs *regs, int trapnr, |
107 | unsigned long error_code, | |
108 | unsigned long fault_addr) | |
d5c8028b EB |
109 | { |
110 | regs->ip = ex_fixup_addr(fixup); | |
111 | ||
112 | WARN_ONCE(1, "Bad FPU state detected at %pB, reinitializing FPU registers.", | |
113 | (void *)instruction_pointer(regs)); | |
114 | ||
115 | __copy_kernel_to_fpregs(&init_fpstate, -1); | |
116 | return true; | |
117 | } | |
118 | EXPORT_SYMBOL_GPL(ex_handler_fprestore); | |
119 | ||
75045f77 | 120 | __visible bool ex_handler_uaccess(const struct exception_table_entry *fixup, |
81fd9c18 JH |
121 | struct pt_regs *regs, int trapnr, |
122 | unsigned long error_code, | |
123 | unsigned long fault_addr) | |
75045f77 | 124 | { |
00c42373 | 125 | WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?"); |
75045f77 JH |
126 | regs->ip = ex_fixup_addr(fixup); |
127 | return true; | |
128 | } | |
129 | EXPORT_SYMBOL(ex_handler_uaccess); | |
130 | ||
80a3e394 | 131 | __visible bool ex_handler_ext(const struct exception_table_entry *fixup, |
81fd9c18 JH |
132 | struct pt_regs *regs, int trapnr, |
133 | unsigned long error_code, | |
134 | unsigned long fault_addr) | |
548acf19 TL |
135 | { |
136 | /* Special hack for uaccess_err */ | |
dfa9a942 | 137 | current->thread.uaccess_err = 1; |
548acf19 TL |
138 | regs->ip = ex_fixup_addr(fixup); |
139 | return true; | |
140 | } | |
141 | EXPORT_SYMBOL(ex_handler_ext); | |
142 | ||
80a3e394 | 143 | __visible bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup, |
81fd9c18 JH |
144 | struct pt_regs *regs, int trapnr, |
145 | unsigned long error_code, | |
146 | unsigned long fault_addr) | |
fbd70437 | 147 | { |
d75f773c | 148 | if (pr_warn_once("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", |
81c2949f BP |
149 | (unsigned int)regs->cx, regs->ip, (void *)regs->ip)) |
150 | show_stack_regs(regs); | |
fbd70437 AL |
151 | |
152 | /* Pretend that the read succeeded and returned 0. */ | |
153 | regs->ip = ex_fixup_addr(fixup); | |
154 | regs->ax = 0; | |
155 | regs->dx = 0; | |
156 | return true; | |
157 | } | |
158 | EXPORT_SYMBOL(ex_handler_rdmsr_unsafe); | |
159 | ||
80a3e394 | 160 | __visible bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup, |
81fd9c18 JH |
161 | struct pt_regs *regs, int trapnr, |
162 | unsigned long error_code, | |
163 | unsigned long fault_addr) | |
fbd70437 | 164 | { |
d75f773c | 165 | if (pr_warn_once("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", |
81c2949f BP |
166 | (unsigned int)regs->cx, (unsigned int)regs->dx, |
167 | (unsigned int)regs->ax, regs->ip, (void *)regs->ip)) | |
168 | show_stack_regs(regs); | |
fbd70437 AL |
169 | |
170 | /* Pretend that the write succeeded. */ | |
171 | regs->ip = ex_fixup_addr(fixup); | |
172 | return true; | |
173 | } | |
174 | EXPORT_SYMBOL(ex_handler_wrmsr_unsafe); | |
175 | ||
80a3e394 | 176 | __visible bool ex_handler_clear_fs(const struct exception_table_entry *fixup, |
81fd9c18 JH |
177 | struct pt_regs *regs, int trapnr, |
178 | unsigned long error_code, | |
179 | unsigned long fault_addr) | |
45e876f7 AL |
180 | { |
181 | if (static_cpu_has(X86_BUG_NULL_SEG)) | |
182 | asm volatile ("mov %0, %%fs" : : "rm" (__USER_DS)); | |
183 | asm volatile ("mov %0, %%fs" : : "rm" (0)); | |
81fd9c18 | 184 | return ex_handler_default(fixup, regs, trapnr, error_code, fault_addr); |
45e876f7 AL |
185 | } |
186 | EXPORT_SYMBOL(ex_handler_clear_fs); | |
187 | ||
80a3e394 | 188 | __visible bool ex_has_fault_handler(unsigned long ip) |
548acf19 TL |
189 | { |
190 | const struct exception_table_entry *e; | |
191 | ex_handler_t handler; | |
192 | ||
193 | e = search_exception_tables(ip); | |
194 | if (!e) | |
195 | return false; | |
196 | handler = ex_fixup_handler(e); | |
197 | ||
198 | return handler == ex_handler_fault; | |
199 | } | |
200 | ||
81fd9c18 JH |
201 | int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, |
202 | unsigned long fault_addr) | |
548acf19 TL |
203 | { |
204 | const struct exception_table_entry *e; | |
205 | ex_handler_t handler; | |
6d48583b HH |
206 | |
207 | #ifdef CONFIG_PNPBIOS | |
208 | if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) { | |
209 | extern u32 pnp_bios_fault_eip, pnp_bios_fault_esp; | |
210 | extern u32 pnp_bios_is_utter_crap; | |
211 | pnp_bios_is_utter_crap = 1; | |
212 | printk(KERN_CRIT "PNPBIOS fault.. attempting recovery.\n"); | |
213 | __asm__ volatile( | |
214 | "movl %0, %%esp\n\t" | |
215 | "jmp *%1\n\t" | |
216 | : : "g" (pnp_bios_fault_esp), "g" (pnp_bios_fault_eip)); | |
217 | panic("do_trap: can't hit this"); | |
218 | } | |
219 | #endif | |
220 | ||
548acf19 TL |
221 | e = search_exception_tables(regs->ip); |
222 | if (!e) | |
223 | return 0; | |
6d48583b | 224 | |
548acf19 | 225 | handler = ex_fixup_handler(e); |
81fd9c18 | 226 | return handler(e, regs, trapnr, error_code, fault_addr); |
6d48583b | 227 | } |
6a1ea279 | 228 | |
0e861fbb AL |
229 | extern unsigned int early_recursion_flag; |
230 | ||
6a1ea279 | 231 | /* Restricted version used during very early boot */ |
0e861fbb | 232 | void __init early_fixup_exception(struct pt_regs *regs, int trapnr) |
6a1ea279 | 233 | { |
0d0efc07 AL |
234 | /* Ignore early NMIs. */ |
235 | if (trapnr == X86_TRAP_NMI) | |
0e861fbb AL |
236 | return; |
237 | ||
238 | if (early_recursion_flag > 2) | |
239 | goto halt_loop; | |
240 | ||
fc0e81b2 AL |
241 | /* |
242 | * Old CPUs leave the high bits of CS on the stack | |
243 | * undefined. I'm not sure which CPUs do this, but at least | |
244 | * the 486 DX works this way. | |
42b3a4cb | 245 | * Xen pv domains are not using the default __KERNEL_CS. |
fc0e81b2 | 246 | */ |
42b3a4cb | 247 | if (!xen_pv_domain() && regs->cs != __KERNEL_CS) |
0e861fbb | 248 | goto fail; |
0d0efc07 | 249 | |
60a0e203 AL |
250 | /* |
251 | * The full exception fixup machinery is available as soon as | |
252 | * the early IDT is loaded. This means that it is the | |
253 | * responsibility of extable users to either function correctly | |
254 | * when handlers are invoked early or to simply avoid causing | |
255 | * exceptions before they're ready to handle them. | |
256 | * | |
257 | * This is better than filtering which handlers can be used, | |
258 | * because refusing to call a handler here is guaranteed to | |
259 | * result in a hard-to-debug panic. | |
260 | * | |
261 | * Keep in mind that not all vectors actually get here. Early | |
81fd9c18 | 262 | * page faults, for example, are special. |
60a0e203 | 263 | */ |
81fd9c18 | 264 | if (fixup_exception(regs, trapnr, regs->orig_ax, 0)) |
ae7ef45e | 265 | return; |
0e861fbb | 266 | |
8a524f80 PZ |
267 | if (fixup_bug(regs, trapnr)) |
268 | return; | |
269 | ||
0e861fbb AL |
270 | fail: |
271 | early_printk("PANIC: early exception 0x%02x IP %lx:%lx error %lx cr2 0x%lx\n", | |
272 | (unsigned)trapnr, (unsigned long)regs->cs, regs->ip, | |
273 | regs->orig_ax, read_cr2()); | |
274 | ||
275 | show_regs(regs); | |
276 | ||
277 | halt_loop: | |
278 | while (true) | |
279 | halt(); | |
6a1ea279 | 280 | } |