projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
x86, ioapic: Don't call disconnect_bsp_APIC if no APIC present
[linux-2.6-block.git] / arch / x86 / kernel / entry_32.S
CommitLineData
1da177e4 1/*
1da177e4
LT		 2 *
3 * Copyright (C) 1991, 1992 Linus Torvalds
4 */
5
6/*
7 * entry.S contains the system-call and fault low-level handling routines.
8 * This also contains the timer-interrupt handler, as well as all interrupts
9 * and faults that can result in a task-switch.
10 *
11 * NOTE: This code handles signal-recognition, which happens every time
12 * after a timer-interrupt and after each system call.
13 *
14 * I changed all the .align's to 4 (16 byte alignment), as that's faster
15 * on a 486.
16 *
889f21ce 17 * Stack layout in 'syscall_exit':
1da177e4
LT		 18 * ptrace needs to have all regs on the stack.
19 * if the order here is changed, it needs to be
20 * updated in fork.c:copy_process, signal.c:do_signal,
21 * ptrace.c and ptrace.h
22 *
23 * 0(%esp) - %ebx
24 * 4(%esp) - %ecx
25 * 8(%esp) - %edx
26 * C(%esp) - %esi
27 * 10(%esp) - %edi
28 * 14(%esp) - %ebp
29 * 18(%esp) - %eax
30 * 1C(%esp) - %ds
31 * 20(%esp) - %es
464d1a78 32 * 24(%esp) - %fs
ccbeed3a
TH		 33 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS
34 * 2C(%esp) - orig_eax
35 * 30(%esp) - %eip
36 * 34(%esp) - %cs
37 * 38(%esp) - %eflags
38 * 3C(%esp) - %oldesp
39 * 40(%esp) - %oldss
1da177e4
LT		 40 *
41 * "current" is in register %ebx during any slow entries.
42 */
43
1da177e4
LT		 44#include <linux/linkage.h>
45#include <asm/thread_info.h>
55f327fa 46#include <asm/irqflags.h>
1da177e4
LT		 47#include <asm/errno.h>
48#include <asm/segment.h>
49#include <asm/smp.h>
0341c14d 50#include <asm/page_types.h>
1da177e4 51#include <asm/desc.h>
be44d2aa 52#include <asm/percpu.h>
fe7cacc1 53#include <asm/dwarf2.h>
ab68ed98 54#include <asm/processor-flags.h>
395a59d0 55#include <asm/ftrace.h>
9b7dc567 56#include <asm/irq_vectors.h>
1da177e4 57
af0575bb
RM		 58/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
59#include <linux/elf-em.h>
60#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE)
61#define __AUDIT_ARCH_LE 0x40000000
62
63#ifndef CONFIG_AUDITSYSCALL
64#define sysenter_audit syscall_trace_entry
65#define sysexit_audit syscall_exit_work
66#endif
67
139ec7c4
RR		 68/*
69 * We use macros for low-level operations which need to be overridden
70 * for paravirtualization. The following will never clobber any registers:
71 * INTERRUPT_RETURN (aka. "iret")
72 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
d75cd22f 73 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
139ec7c4
RR		 74 *
75 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
76 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
77 * Allowing a register to be clobbered can shrink the paravirt replacement
78 * enough to patch inline, increasing performance.
79 */
80
1da177e4
LT		 81#define nr_syscalls ((syscall_table_size)/4)
82
1da177e4 83#ifdef CONFIG_PREEMPT
139ec7c4 84#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
1da177e4 85#else
139ec7c4 86#define preempt_stop(clobbers)
1da177e4
LT		 87#define resume_kernel restore_nocheck
88#endif
89
55f327fa
IM		 90.macro TRACE_IRQS_IRET
91#ifdef CONFIG_TRACE_IRQFLAGS
ab68ed98 92 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off?
55f327fa
IM		 93 jz 1f
94 TRACE_IRQS_ON
951:
96#endif
97.endm
98
4031ff38
AG		 99#ifdef CONFIG_VM86
100#define resume_userspace_sig check_userspace
101#else
102#define resume_userspace_sig resume_userspace
103#endif
104
ccbeed3a
TH		 105/*
106 * User gs save/restore
107 *
108 * %gs is used for userland TLS and kernel only uses it for stack
109 * canary which is required to be at %gs:20 by gcc. Read the comment
110 * at the top of stackprotector.h for more info.
111 *
112 * Local labels 98 and 99 are used.
113 */
114#ifdef CONFIG_X86_32_LAZY_GS
115
116 /* unfortunately push/pop can't be no-op */
117.macro PUSH_GS
118 pushl $0
119 CFI_ADJUST_CFA_OFFSET 4
120.endm
121.macro POP_GS pop=0
122 addl $(4 + \pop), %esp
123 CFI_ADJUST_CFA_OFFSET -(4 + \pop)
124.endm
125.macro POP_GS_EX
126.endm
127
128 /* all the rest are no-op */
129.macro PTGS_TO_GS
130.endm
131.macro PTGS_TO_GS_EX
132.endm
133.macro GS_TO_REG reg
134.endm
135.macro REG_TO_PTGS reg
136.endm
137.macro SET_KERNEL_GS reg
138.endm
139
140#else /* CONFIG_X86_32_LAZY_GS */
141
142.macro PUSH_GS
143 pushl %gs
144 CFI_ADJUST_CFA_OFFSET 4
145 /*CFI_REL_OFFSET gs, 0*/
146.endm
147
148.macro POP_GS pop=0
14998: popl %gs
150 CFI_ADJUST_CFA_OFFSET -4
151 /*CFI_RESTORE gs*/
152 .if \pop <> 0
153 add $\pop, %esp
154 CFI_ADJUST_CFA_OFFSET -\pop
155 .endif
156.endm
157.macro POP_GS_EX
158.pushsection .fixup, "ax"
15999: movl $0, (%esp)
160 jmp 98b
161.section __ex_table, "a"
162 .align 4
163 .long 98b, 99b
164.popsection
165.endm
166
167.macro PTGS_TO_GS
16898: mov PT_GS(%esp), %gs
169.endm
170.macro PTGS_TO_GS_EX
171.pushsection .fixup, "ax"
17299: movl $0, PT_GS(%esp)
173 jmp 98b
174.section __ex_table, "a"
175 .align 4
176 .long 98b, 99b
177.popsection
178.endm
179
180.macro GS_TO_REG reg
181 movl %gs, \reg
182 /*CFI_REGISTER gs, \reg*/
183.endm
184.macro REG_TO_PTGS reg
185 movl \reg, PT_GS(%esp)
186 /*CFI_REL_OFFSET gs, PT_GS*/
187.endm
188.macro SET_KERNEL_GS reg
60a5317f 189 movl $(__KERNEL_STACK_CANARY), \reg
ccbeed3a
TH		 190 movl \reg, %gs
191.endm
192
193#endif /* CONFIG_X86_32_LAZY_GS */
194
f0d96110
TH		 195.macro SAVE_ALL
196 cld
ccbeed3a 197 PUSH_GS
f0d96110
TH		 198 pushl %fs
199 CFI_ADJUST_CFA_OFFSET 4
200 /*CFI_REL_OFFSET fs, 0;*/
201 pushl %es
202 CFI_ADJUST_CFA_OFFSET 4
203 /*CFI_REL_OFFSET es, 0;*/
204 pushl %ds
205 CFI_ADJUST_CFA_OFFSET 4
206 /*CFI_REL_OFFSET ds, 0;*/
207 pushl %eax
208 CFI_ADJUST_CFA_OFFSET 4
209 CFI_REL_OFFSET eax, 0
210 pushl %ebp
211 CFI_ADJUST_CFA_OFFSET 4
212 CFI_REL_OFFSET ebp, 0
213 pushl %edi
214 CFI_ADJUST_CFA_OFFSET 4
215 CFI_REL_OFFSET edi, 0
216 pushl %esi
217 CFI_ADJUST_CFA_OFFSET 4
218 CFI_REL_OFFSET esi, 0
219 pushl %edx
220 CFI_ADJUST_CFA_OFFSET 4
221 CFI_REL_OFFSET edx, 0
222 pushl %ecx
223 CFI_ADJUST_CFA_OFFSET 4
224 CFI_REL_OFFSET ecx, 0
225 pushl %ebx
226 CFI_ADJUST_CFA_OFFSET 4
227 CFI_REL_OFFSET ebx, 0
228 movl $(__USER_DS), %edx
229 movl %edx, %ds
230 movl %edx, %es
231 movl $(__KERNEL_PERCPU), %edx
464d1a78 232 movl %edx, %fs
ccbeed3a 233 SET_KERNEL_GS %edx
f0d96110 234.endm
1da177e4 235
f0d96110
TH		 236.macro RESTORE_INT_REGS
237 popl %ebx
238 CFI_ADJUST_CFA_OFFSET -4
239 CFI_RESTORE ebx
240 popl %ecx
241 CFI_ADJUST_CFA_OFFSET -4
242 CFI_RESTORE ecx
243 popl %edx
244 CFI_ADJUST_CFA_OFFSET -4
245 CFI_RESTORE edx
246 popl %esi
247 CFI_ADJUST_CFA_OFFSET -4
248 CFI_RESTORE esi
249 popl %edi
250 CFI_ADJUST_CFA_OFFSET -4
251 CFI_RESTORE edi
252 popl %ebp
253 CFI_ADJUST_CFA_OFFSET -4
254 CFI_RESTORE ebp
255 popl %eax
256 CFI_ADJUST_CFA_OFFSET -4
fe7cacc1 257 CFI_RESTORE eax
f0d96110 258.endm
1da177e4 259
ccbeed3a 260.macro RESTORE_REGS pop=0
f0d96110
TH		 261 RESTORE_INT_REGS
2621: popl %ds
263 CFI_ADJUST_CFA_OFFSET -4
264 /*CFI_RESTORE ds;*/
2652: popl %es
266 CFI_ADJUST_CFA_OFFSET -4
267 /*CFI_RESTORE es;*/
2683: popl %fs
269 CFI_ADJUST_CFA_OFFSET -4
270 /*CFI_RESTORE fs;*/
ccbeed3a 271 POP_GS \pop
f0d96110
TH		 272.pushsection .fixup, "ax"
2734: movl $0, (%esp)
274 jmp 1b
2755: movl $0, (%esp)
276 jmp 2b
2776: movl $0, (%esp)
278 jmp 3b
279.section __ex_table, "a"
280 .align 4
281 .long 1b, 4b
282 .long 2b, 5b
283 .long 3b, 6b
f95d47ca 284.popsection
ccbeed3a 285 POP_GS_EX
f0d96110 286.endm
1da177e4 287
f0d96110
TH		 288.macro RING0_INT_FRAME
289 CFI_STARTPROC simple
290 CFI_SIGNAL_FRAME
291 CFI_DEF_CFA esp, 3*4
292 /*CFI_OFFSET cs, -2*4;*/
fe7cacc1 293 CFI_OFFSET eip, -3*4
f0d96110 294.endm
fe7cacc1 295
f0d96110
TH		 296.macro RING0_EC_FRAME
297 CFI_STARTPROC simple
298 CFI_SIGNAL_FRAME
299 CFI_DEF_CFA esp, 4*4
300 /*CFI_OFFSET cs, -2*4;*/
fe7cacc1 301 CFI_OFFSET eip, -3*4
f0d96110 302.endm
fe7cacc1 303
f0d96110
TH		 304.macro RING0_PTREGS_FRAME
305 CFI_STARTPROC simple
306 CFI_SIGNAL_FRAME
307 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX
308 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/
309 CFI_OFFSET eip, PT_EIP-PT_OLDESP
310 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/
311 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/
312 CFI_OFFSET eax, PT_EAX-PT_OLDESP
313 CFI_OFFSET ebp, PT_EBP-PT_OLDESP
314 CFI_OFFSET edi, PT_EDI-PT_OLDESP
315 CFI_OFFSET esi, PT_ESI-PT_OLDESP
316 CFI_OFFSET edx, PT_EDX-PT_OLDESP
317 CFI_OFFSET ecx, PT_ECX-PT_OLDESP
eb5b7b9d 318 CFI_OFFSET ebx, PT_EBX-PT_OLDESP
f0d96110 319.endm
1da177e4
LT		 320
321ENTRY(ret_from_fork)
fe7cacc1 322 CFI_STARTPROC
1da177e4 323 pushl %eax
25d7dfda 324 CFI_ADJUST_CFA_OFFSET 4
1da177e4
LT		 325 call schedule_tail
326 GET_THREAD_INFO(%ebp)
327 popl %eax
fe7cacc1 328 CFI_ADJUST_CFA_OFFSET -4
47a5c6fa
LT		 329 pushl $0x0202 # Reset kernel eflags
330 CFI_ADJUST_CFA_OFFSET 4
331 popfl
332 CFI_ADJUST_CFA_OFFSET -4
1da177e4 333 jmp syscall_exit
fe7cacc1 334 CFI_ENDPROC
47a55cd7 335END(ret_from_fork)
1da177e4
LT		 336
337/*
338 * Return to user mode is not as complex as all this looks,
339 * but we want the default path for a system call return to
340 * go as quickly as possible which is why some of this is
341 * less clear than it otherwise should be.
342 */
343
344 # userspace resumption stub bypassing syscall exit tracing
345 ALIGN
fe7cacc1 346 RING0_PTREGS_FRAME
1da177e4 347ret_from_exception:
139ec7c4 348 preempt_stop(CLBR_ANY)
1da177e4
LT		 349ret_from_intr:
350 GET_THREAD_INFO(%ebp)
4031ff38 351check_userspace:
eb5b7b9d
JF		 352 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
353 movb PT_CS(%esp), %al
ab68ed98 354 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
78be3706
RR		 355 cmpl $USER_RPL, %eax
356 jb resume_kernel # not returning to v8086 or userspace
f95d47ca 357
1da177e4 358ENTRY(resume_userspace)
c7e872e7 359 LOCKDEP_SYS_EXIT
139ec7c4 360 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
1da177e4
LT		 361 # setting need_resched or sigpending
362 # between sampling and the iret
e32e58a9 363 TRACE_IRQS_OFF
1da177e4
LT		 364 movl TI_flags(%ebp), %ecx
365 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
366 # int/exception return?
367 jne work_pending
368 jmp restore_all
47a55cd7 369END(ret_from_exception)
1da177e4
LT		 370
371#ifdef CONFIG_PREEMPT
372ENTRY(resume_kernel)
139ec7c4 373 DISABLE_INTERRUPTS(CLBR_ANY)
1da177e4
LT		 374 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
375 jnz restore_nocheck
376need_resched:
377 movl TI_flags(%ebp), %ecx # need_resched set ?
378 testb $_TIF_NEED_RESCHED, %cl
379 jz restore_all
ab68ed98 380 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ?
1da177e4
LT		 381 jz restore_all
382 call preempt_schedule_irq
383 jmp need_resched
47a55cd7 384END(resume_kernel)
1da177e4 385#endif
fe7cacc1 386 CFI_ENDPROC
1da177e4
LT		 387
388/* SYSENTER_RETURN points to after the "sysenter" instruction in
389 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
390
391 # sysenter call handler stub
0aa97fb2 392ENTRY(ia32_sysenter_target)
fe7cacc1 393 CFI_STARTPROC simple
adf14236 394 CFI_SIGNAL_FRAME
fe7cacc1
JB		 395 CFI_DEF_CFA esp, 0
396 CFI_REGISTER esp, ebp
faca6227 397 movl TSS_sysenter_sp0(%esp),%esp
1da177e4 398sysenter_past_esp:
55f327fa 399 /*
d93c870b
JF		 400 * Interrupts are disabled here, but we can't trace it until
401 * enough kernel state to call TRACE_IRQS_OFF can be called - but
402 * we immediately enable interrupts at that point anyway.
55f327fa 403 */
1da177e4 404 pushl $(__USER_DS)
fe7cacc1
JB		 405 CFI_ADJUST_CFA_OFFSET 4
406 /*CFI_REL_OFFSET ss, 0*/
1da177e4 407 pushl %ebp
fe7cacc1
JB		 408 CFI_ADJUST_CFA_OFFSET 4
409 CFI_REL_OFFSET esp, 0
1da177e4 410 pushfl
d93c870b 411 orl $X86_EFLAGS_IF, (%esp)
fe7cacc1 412 CFI_ADJUST_CFA_OFFSET 4
1da177e4 413 pushl $(__USER_CS)
fe7cacc1
JB		 414 CFI_ADJUST_CFA_OFFSET 4
415 /*CFI_REL_OFFSET cs, 0*/
e6e5494c
IM		 416 /*
417 * Push current_thread_info()->sysenter_return to the stack.
418 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
419 * pushed above; +8 corresponds to copy_thread's esp0 setting.
420 */
421 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
fe7cacc1
JB		 422 CFI_ADJUST_CFA_OFFSET 4
423 CFI_REL_OFFSET eip, 0
1da177e4 424
d93c870b
JF		 425 pushl %eax
426 CFI_ADJUST_CFA_OFFSET 4
427 SAVE_ALL
428 ENABLE_INTERRUPTS(CLBR_NONE)
429
1da177e4
LT		 430/*
431 * Load the potential sixth argument from user stack.
432 * Careful about security.
433 */
434 cmpl $__PAGE_OFFSET-3,%ebp
435 jae syscall_fault
4361: movl (%ebp),%ebp
d93c870b 437 movl %ebp,PT_EBP(%esp)
1da177e4
LT		 438.section __ex_table,"a"
439 .align 4
440 .long 1b,syscall_fault
441.previous
442
1da177e4
LT		 443 GET_THREAD_INFO(%ebp)
444
88200bc2 445 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
af0575bb
RM		 446 jnz sysenter_audit
447sysenter_do_call:
1da177e4
LT		 448 cmpl $(nr_syscalls), %eax
449 jae syscall_badsys
450 call *sys_call_table(,%eax,4)
eb5b7b9d 451 movl %eax,PT_EAX(%esp)
c7e872e7 452 LOCKDEP_SYS_EXIT
42c24fa2 453 DISABLE_INTERRUPTS(CLBR_ANY)
55f327fa 454 TRACE_IRQS_OFF
1da177e4 455 movl TI_flags(%ebp), %ecx
88200bc2 456 testl $_TIF_ALLWORK_MASK, %ecx
af0575bb
RM		 457 jne sysexit_audit
458sysenter_exit:
1da177e4 459/* if something modifies registers it must also disable sysexit */
eb5b7b9d
JF		 460 movl PT_EIP(%esp), %edx
461 movl PT_OLDESP(%esp), %ecx
1da177e4 462 xorl %ebp,%ebp
55f327fa 463 TRACE_IRQS_ON
464d1a78 4641: mov PT_FS(%esp), %fs
ccbeed3a 465 PTGS_TO_GS
d75cd22f 466 ENABLE_INTERRUPTS_SYSEXIT
af0575bb
RM		 467
468#ifdef CONFIG_AUDITSYSCALL
469sysenter_audit:
88200bc2 470 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
af0575bb
RM		 471 jnz syscall_trace_entry
472 addl $4,%esp
473 CFI_ADJUST_CFA_OFFSET -4
474 /* %esi already in 8(%esp) 6th arg: 4th syscall arg */
475 /* %edx already in 4(%esp) 5th arg: 3rd syscall arg */
476 /* %ecx already in 0(%esp) 4th arg: 2nd syscall arg */
477 movl %ebx,%ecx /* 3rd arg: 1st syscall arg */
478 movl %eax,%edx /* 2nd arg: syscall number */
479 movl $AUDIT_ARCH_I386,%eax /* 1st arg: audit arch */
480 call audit_syscall_entry
481 pushl %ebx
482 CFI_ADJUST_CFA_OFFSET 4
483 movl PT_EAX(%esp),%eax /* reload syscall number */
484 jmp sysenter_do_call
485
486sysexit_audit:
88200bc2 487 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
af0575bb
RM		 488 jne syscall_exit_work
489 TRACE_IRQS_ON
490 ENABLE_INTERRUPTS(CLBR_ANY)
491 movl %eax,%edx /* second arg, syscall return value */
492 cmpl $0,%eax /* is it < 0? */
493 setl %al /* 1 if so, 0 if not */
494 movzbl %al,%eax /* zero-extend that */
495 inc %eax /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
496 call audit_syscall_exit
497 DISABLE_INTERRUPTS(CLBR_ANY)
498 TRACE_IRQS_OFF
499 movl TI_flags(%ebp), %ecx
88200bc2 500 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
af0575bb
RM		 501 jne syscall_exit_work
502 movl PT_EAX(%esp),%eax /* reload syscall return value */
503 jmp sysenter_exit
504#endif
505
fe7cacc1 506 CFI_ENDPROC
f95d47ca 507.pushsection .fixup,"ax"
464d1a78 5082: movl $0,PT_FS(%esp)
f95d47ca
JF		 509 jmp 1b
510.section __ex_table,"a"
511 .align 4
512 .long 1b,2b
513.popsection
ccbeed3a 514 PTGS_TO_GS_EX
0aa97fb2 515ENDPROC(ia32_sysenter_target)
1da177e4
LT		 516
517 # system call handler stub
518ENTRY(system_call)
fe7cacc1 519 RING0_INT_FRAME # can't unwind into user space anyway
1da177e4 520 pushl %eax # save orig_eax
fe7cacc1 521 CFI_ADJUST_CFA_OFFSET 4
1da177e4
LT		 522 SAVE_ALL
523 GET_THREAD_INFO(%ebp)
ed75e8d5 524 # system call tracing in operation / emulation
88200bc2 525 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
1da177e4
LT		 526 jnz syscall_trace_entry
527 cmpl $(nr_syscalls), %eax
528 jae syscall_badsys
529syscall_call:
530 call *sys_call_table(,%eax,4)
eb5b7b9d 531 movl %eax,PT_EAX(%esp) # store the return value
1da177e4 532syscall_exit:
c7e872e7 533 LOCKDEP_SYS_EXIT
139ec7c4 534 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
1da177e4
LT		 535 # setting need_resched or sigpending
536 # between sampling and the iret
55f327fa 537 TRACE_IRQS_OFF
1da177e4 538 movl TI_flags(%ebp), %ecx
88200bc2 539 testl $_TIF_ALLWORK_MASK, %ecx # current->work
1da177e4
LT		 540 jne syscall_exit_work
541
542restore_all:
eb5b7b9d
JF		 543 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
544 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
5df24082
SS		 545 # are returning to the kernel.
546 # See comments in process.c:copy_thread() for details.
eb5b7b9d
JF		 547 movb PT_OLDSS(%esp), %ah
548 movb PT_CS(%esp), %al
ab68ed98 549 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
78be3706 550 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
fe7cacc1 551 CFI_REMEMBER_STATE
1da177e4
LT		 552 je ldt_ss # returning to user-space with LDT SS
553restore_nocheck:
55f327fa
IM		 554 TRACE_IRQS_IRET
555restore_nocheck_notrace:
ccbeed3a 556 RESTORE_REGS 4 # skip orig_eax/error_code
fe7cacc1 557 CFI_ADJUST_CFA_OFFSET -4
f7f3d791 558irq_return:
3701d863 559 INTERRUPT_RETURN
1da177e4 560.section .fixup,"ax"
90e9f536 561ENTRY(iret_exc)
a879cbbb
LT		 562 pushl $0 # no error code
563 pushl $do_iret_error
564 jmp error_code
1da177e4
LT		 565.previous
566.section __ex_table,"a"
567 .align 4
3701d863 568 .long irq_return,iret_exc
1da177e4
LT		 569.previous
570
fe7cacc1 571 CFI_RESTORE_STATE
1da177e4 572ldt_ss:
eb5b7b9d 573 larl PT_OLDSS(%esp), %eax
1da177e4
LT		 574 jnz restore_nocheck
575 testl $0x00400000, %eax # returning to 32bit stack?
576 jnz restore_nocheck # allright, normal return
d3561b7f
RR		 577
578#ifdef CONFIG_PARAVIRT
579 /*
580 * The kernel can't run on a non-flat stack if paravirt mode
581 * is active. Rather than try to fixup the high bits of
582 * ESP, bypass this code entirely. This may break DOSemu
583 * and/or Wine support in a paravirt VM, although the option
584 * is still available to implement the setting of the high
585 * 16-bits in the INTERRUPT_RETURN paravirt-op.
586 */
93b1eab3 587 cmpl $0, pv_info+PARAVIRT_enabled
d3561b7f
RR		 588 jne restore_nocheck
589#endif
590
1da177e4
LT		 591 /* If returning to userspace with 16bit stack,
592 * try to fix the higher word of ESP, as the CPU
593 * won't restore it.
594 * This is an "official" bug of all the x86-compatible
595 * CPUs, which we can try to work around to make
596 * dosemu and wine happy. */
eb5b7b9d 597 movl PT_OLDESP(%esp), %eax
be44d2aa
SS		 598 movl %esp, %edx
599 call patch_espfix_desc
600 pushl $__ESPFIX_SS
601 CFI_ADJUST_CFA_OFFSET 4
602 pushl %eax
603 CFI_ADJUST_CFA_OFFSET 4
139ec7c4 604 DISABLE_INTERRUPTS(CLBR_EAX)
55f327fa 605 TRACE_IRQS_OFF
be44d2aa
SS		 606 lss (%esp), %esp
607 CFI_ADJUST_CFA_OFFSET -8
608 jmp restore_nocheck
fe7cacc1 609 CFI_ENDPROC
47a55cd7 610ENDPROC(system_call)
1da177e4
LT		 611
612 # perform work that needs to be done immediately before resumption
613 ALIGN
fe7cacc1 614 RING0_PTREGS_FRAME # can't unwind into user space anyway
1da177e4
LT		 615work_pending:
616 testb $_TIF_NEED_RESCHED, %cl
617 jz work_notifysig
618work_resched:
619 call schedule
c7e872e7 620 LOCKDEP_SYS_EXIT
139ec7c4 621 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
1da177e4
LT		 622 # setting need_resched or sigpending
623 # between sampling and the iret
55f327fa 624 TRACE_IRQS_OFF
1da177e4
LT		 625 movl TI_flags(%ebp), %ecx
626 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
627 # than syscall tracing?
628 jz restore_all
629 testb $_TIF_NEED_RESCHED, %cl
630 jnz work_resched
631
632work_notifysig: # deal with pending signals and
633 # notify-resume requests
74b47a78 634#ifdef CONFIG_VM86
ab68ed98 635 testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
1da177e4
LT		 636 movl %esp, %eax
637 jne work_notifysig_v86 # returning to kernel-space or
638 # vm86-space
639 xorl %edx, %edx
640 call do_notify_resume
4031ff38 641 jmp resume_userspace_sig
1da177e4
LT		 642
643 ALIGN
644work_notifysig_v86:
645 pushl %ecx # save ti_flags for do_notify_resume
fe7cacc1 646 CFI_ADJUST_CFA_OFFSET 4
1da177e4
LT		 647 call save_v86_state # %eax contains pt_regs pointer
648 popl %ecx
fe7cacc1 649 CFI_ADJUST_CFA_OFFSET -4
1da177e4 650 movl %eax, %esp
74b47a78
JK		 651#else
652 movl %esp, %eax
653#endif
1da177e4
LT		 654 xorl %edx, %edx
655 call do_notify_resume
4031ff38 656 jmp resume_userspace_sig
47a55cd7 657END(work_pending)
1da177e4
LT		 658
659 # perform syscall exit tracing
660 ALIGN
661syscall_trace_entry:
eb5b7b9d 662 movl $-ENOSYS,PT_EAX(%esp)
1da177e4 663 movl %esp, %eax
d4d67150
RM		 664 call syscall_trace_enter
665 /* What it returned is what we'll actually use. */
1da177e4
LT		 666 cmpl $(nr_syscalls), %eax
667 jnae syscall_call
668 jmp syscall_exit
47a55cd7 669END(syscall_trace_entry)
1da177e4
LT		 670
671 # perform syscall exit tracing
672 ALIGN
673syscall_exit_work:
88200bc2 674 testl $_TIF_WORK_SYSCALL_EXIT, %ecx
1da177e4 675 jz work_pending
55f327fa 676 TRACE_IRQS_ON
d4d67150 677 ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call
1da177e4
LT		 678 # schedule() instead
679 movl %esp, %eax
d4d67150 680 call syscall_trace_leave
1da177e4 681 jmp resume_userspace
47a55cd7 682END(syscall_exit_work)
fe7cacc1 683 CFI_ENDPROC
1da177e4 684
fe7cacc1 685 RING0_INT_FRAME # can't unwind into user space anyway
1da177e4 686syscall_fault:
1da177e4 687 GET_THREAD_INFO(%ebp)
eb5b7b9d 688 movl $-EFAULT,PT_EAX(%esp)
1da177e4 689 jmp resume_userspace
47a55cd7 690END(syscall_fault)
1da177e4 691
1da177e4 692syscall_badsys:
eb5b7b9d 693 movl $-ENOSYS,PT_EAX(%esp)
1da177e4 694 jmp resume_userspace
47a55cd7 695END(syscall_badsys)
fe7cacc1 696 CFI_ENDPROC
1da177e4 697
253f29a4
BG		 698/*
699 * System calls that need a pt_regs pointer.
700 */
701#define PTREGSCALL(name) \
702 ALIGN; \
703ptregs_##name: \
704 leal 4(%esp),%eax; \
705 jmp sys_##name;
706
707PTREGSCALL(iopl)
708PTREGSCALL(fork)
709PTREGSCALL(clone)
710PTREGSCALL(vfork)
711PTREGSCALL(execve)
712PTREGSCALL(sigaltstack)
713PTREGSCALL(sigreturn)
714PTREGSCALL(rt_sigreturn)
715PTREGSCALL(vm86)
716PTREGSCALL(vm86old)
717
f0d96110
TH		 718.macro FIXUP_ESPFIX_STACK
719 /* since we are on a wrong stack, we cant make it a C code :( */
720 PER_CPU(gdt_page, %ebx)
721 GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah)
722 addl %esp, %eax
723 pushl $__KERNEL_DS
724 CFI_ADJUST_CFA_OFFSET 4
725 pushl %eax
726 CFI_ADJUST_CFA_OFFSET 4
727 lss (%esp), %esp
728 CFI_ADJUST_CFA_OFFSET -8
729.endm
730.macro UNWIND_ESPFIX_STACK
731 movl %ss, %eax
732 /* see if on espfix stack */
733 cmpw $__ESPFIX_SS, %ax
734 jne 27f
735 movl $__KERNEL_DS, %eax
736 movl %eax, %ds
737 movl %eax, %es
738 /* switch to normal stack */
739 FIXUP_ESPFIX_STACK
74027:
741.endm
1da177e4
LT		 742
743/*
b7c6244f
PA		 744 * Build the entry stubs and pointer table with some assembler magic.
745 * We pack 7 stubs into a single 32-byte chunk, which will fit in a
746 * single cache line on all modern x86 implementations.
1da177e4 747 */
4687518c 748.section .init.rodata,"a"
1da177e4
LT		 749ENTRY(interrupt)
750.text
b7c6244f
PA		 751 .p2align 5
752 .p2align CONFIG_X86_L1_CACHE_SHIFT
1da177e4 753ENTRY(irq_entries_start)
fe7cacc1 754 RING0_INT_FRAME
4687518c 755vector=FIRST_EXTERNAL_VECTOR
b7c6244f
PA		 756.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
757 .balign 32
758 .rept 7
759 .if vector < NR_VECTORS
8665596e 760 .if vector <> FIRST_EXTERNAL_VECTOR
fe7cacc1 761 CFI_ADJUST_CFA_OFFSET -4
b7c6244f
PA		 762 .endif
7631: pushl $(~vector+0x80) /* Note: always in signed byte range */
fe7cacc1 764 CFI_ADJUST_CFA_OFFSET 4
8665596e 765 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
b7c6244f
PA		 766 jmp 2f
767 .endif
768 .previous
1da177e4 769 .long 1b
b7c6244f 770 .text
1da177e4 771vector=vector+1
b7c6244f
PA		 772 .endif
773 .endr
7742: jmp common_interrupt
1da177e4 775.endr
47a55cd7
JB		 776END(irq_entries_start)
777
778.previous
779END(interrupt)
780.previous
1da177e4 781
55f327fa
IM		 782/*
783 * the CPU automatically disables interrupts when executing an IRQ vector,
784 * so IRQ-flags tracing has to follow that:
785 */
b7c6244f 786 .p2align CONFIG_X86_L1_CACHE_SHIFT
1da177e4 787common_interrupt:
b7c6244f 788 addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */
1da177e4 789 SAVE_ALL
55f327fa 790 TRACE_IRQS_OFF
1da177e4
LT		 791 movl %esp,%eax
792 call do_IRQ
793 jmp ret_from_intr
47a55cd7 794ENDPROC(common_interrupt)
fe7cacc1 795 CFI_ENDPROC
1da177e4 796
02cf94c3 797#define BUILD_INTERRUPT3(name, nr, fn) \
1da177e4 798ENTRY(name) \
fe7cacc1 799 RING0_INT_FRAME; \
19eadf98 800 pushl $~(nr); \
fe7cacc1
JB		 801 CFI_ADJUST_CFA_OFFSET 4; \
802 SAVE_ALL; \
55f327fa 803 TRACE_IRQS_OFF \
1da177e4 804 movl %esp,%eax; \
02cf94c3 805 call fn; \
55f327fa 806 jmp ret_from_intr; \
47a55cd7
JB		 807 CFI_ENDPROC; \
808ENDPROC(name)
1da177e4 809
02cf94c3
TH		 810#define BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(name, nr, smp_##name)
811
1da177e4 812/* The include is where all of the SMP etc. interrupts come from */
1164dd00 813#include <asm/entry_arch.h>
1da177e4 814
1da177e4 815ENTRY(coprocessor_error)
fe7cacc1 816 RING0_INT_FRAME
1da177e4 817 pushl $0
fe7cacc1 818 CFI_ADJUST_CFA_OFFSET 4
1da177e4 819 pushl $do_coprocessor_error
fe7cacc1 820 CFI_ADJUST_CFA_OFFSET 4
1da177e4 821 jmp error_code
fe7cacc1 822 CFI_ENDPROC
47a55cd7 823END(coprocessor_error)
1da177e4
LT		 824
825ENTRY(simd_coprocessor_error)
fe7cacc1 826 RING0_INT_FRAME
1da177e4 827 pushl $0
fe7cacc1 828 CFI_ADJUST_CFA_OFFSET 4
1da177e4 829 pushl $do_simd_coprocessor_error
fe7cacc1 830 CFI_ADJUST_CFA_OFFSET 4
1da177e4 831 jmp error_code
fe7cacc1 832 CFI_ENDPROC
47a55cd7 833END(simd_coprocessor_error)
1da177e4
LT		 834
835ENTRY(device_not_available)
fe7cacc1 836 RING0_INT_FRAME
1da177e4 837 pushl $-1 # mark this as an int
fe7cacc1 838 CFI_ADJUST_CFA_OFFSET 4
7643e9b9 839 pushl $do_device_not_available
fe7cacc1 840 CFI_ADJUST_CFA_OFFSET 4
7643e9b9 841 jmp error_code
fe7cacc1 842 CFI_ENDPROC
47a55cd7 843END(device_not_available)
1da177e4 844
d3561b7f
RR		 845#ifdef CONFIG_PARAVIRT
846ENTRY(native_iret)
3701d863 847 iret
d3561b7f
RR		 848.section __ex_table,"a"
849 .align 4
3701d863 850 .long native_iret, iret_exc
d3561b7f 851.previous
47a55cd7 852END(native_iret)
d3561b7f 853
d75cd22f 854ENTRY(native_irq_enable_sysexit)
d3561b7f
RR		 855 sti
856 sysexit
d75cd22f 857END(native_irq_enable_sysexit)
d3561b7f
RR		 858#endif
859
1da177e4 860ENTRY(overflow)
fe7cacc1 861 RING0_INT_FRAME
1da177e4 862 pushl $0
fe7cacc1 863 CFI_ADJUST_CFA_OFFSET 4
1da177e4 864 pushl $do_overflow
fe7cacc1 865 CFI_ADJUST_CFA_OFFSET 4
1da177e4 866 jmp error_code
fe7cacc1 867 CFI_ENDPROC
47a55cd7 868END(overflow)
1da177e4
LT		 869
870ENTRY(bounds)
fe7cacc1 871 RING0_INT_FRAME
1da177e4 872 pushl $0
fe7cacc1 873 CFI_ADJUST_CFA_OFFSET 4
1da177e4 874 pushl $do_bounds
fe7cacc1 875 CFI_ADJUST_CFA_OFFSET 4
1da177e4 876 jmp error_code
fe7cacc1 877 CFI_ENDPROC
47a55cd7 878END(bounds)
1da177e4
LT		 879
880ENTRY(invalid_op)
fe7cacc1 881 RING0_INT_FRAME
1da177e4 882 pushl $0
fe7cacc1 883 CFI_ADJUST_CFA_OFFSET 4
1da177e4 884 pushl $do_invalid_op
fe7cacc1 885 CFI_ADJUST_CFA_OFFSET 4
1da177e4 886 jmp error_code
fe7cacc1 887 CFI_ENDPROC
47a55cd7 888END(invalid_op)
1da177e4
LT		 889
890ENTRY(coprocessor_segment_overrun)
fe7cacc1 891 RING0_INT_FRAME
1da177e4 892 pushl $0
fe7cacc1 893 CFI_ADJUST_CFA_OFFSET 4
1da177e4 894 pushl $do_coprocessor_segment_overrun
fe7cacc1 895 CFI_ADJUST_CFA_OFFSET 4
1da177e4 896 jmp error_code
fe7cacc1 897 CFI_ENDPROC
47a55cd7 898END(coprocessor_segment_overrun)
1da177e4
LT		 899
900ENTRY(invalid_TSS)
fe7cacc1 901 RING0_EC_FRAME
1da177e4 902 pushl $do_invalid_TSS
fe7cacc1 903 CFI_ADJUST_CFA_OFFSET 4
1da177e4 904 jmp error_code
fe7cacc1 905 CFI_ENDPROC
47a55cd7 906END(invalid_TSS)
1da177e4
LT		 907
908ENTRY(segment_not_present)
fe7cacc1 909 RING0_EC_FRAME
1da177e4 910 pushl $do_segment_not_present
fe7cacc1 911 CFI_ADJUST_CFA_OFFSET 4
1da177e4 912 jmp error_code
fe7cacc1 913 CFI_ENDPROC
47a55cd7 914END(segment_not_present)
1da177e4
LT		 915
916ENTRY(stack_segment)
fe7cacc1 917 RING0_EC_FRAME
1da177e4 918 pushl $do_stack_segment
fe7cacc1 919 CFI_ADJUST_CFA_OFFSET 4
1da177e4 920 jmp error_code
fe7cacc1 921 CFI_ENDPROC
47a55cd7 922END(stack_segment)
1da177e4 923
1da177e4 924ENTRY(alignment_check)
fe7cacc1 925 RING0_EC_FRAME
1da177e4 926 pushl $do_alignment_check
fe7cacc1 927 CFI_ADJUST_CFA_OFFSET 4
1da177e4 928 jmp error_code
fe7cacc1 929 CFI_ENDPROC
47a55cd7 930END(alignment_check)
1da177e4 931
d28c4393
P		 932ENTRY(divide_error)
933 RING0_INT_FRAME
934 pushl $0 # no error code
935 CFI_ADJUST_CFA_OFFSET 4
936 pushl $do_divide_error
fe7cacc1 937 CFI_ADJUST_CFA_OFFSET 4
1da177e4 938 jmp error_code
fe7cacc1 939 CFI_ENDPROC
47a55cd7 940END(divide_error)
1da177e4
LT		 941
942#ifdef CONFIG_X86_MCE
943ENTRY(machine_check)
fe7cacc1 944 RING0_INT_FRAME
1da177e4 945 pushl $0
fe7cacc1 946 CFI_ADJUST_CFA_OFFSET 4
d2f6f7ae 947 pushl machine_check_vector
fe7cacc1 948 CFI_ADJUST_CFA_OFFSET 4
1da177e4 949 jmp error_code
fe7cacc1 950 CFI_ENDPROC
47a55cd7 951END(machine_check)
1da177e4
LT		 952#endif
953
954ENTRY(spurious_interrupt_bug)
fe7cacc1 955 RING0_INT_FRAME
1da177e4 956 pushl $0
fe7cacc1 957 CFI_ADJUST_CFA_OFFSET 4
1da177e4 958 pushl $do_spurious_interrupt_bug
fe7cacc1 959 CFI_ADJUST_CFA_OFFSET 4
1da177e4 960 jmp error_code
fe7cacc1 961 CFI_ENDPROC
47a55cd7 962END(spurious_interrupt_bug)
1da177e4 963
02ba1a32
AK		 964ENTRY(kernel_thread_helper)
965 pushl $0 # fake return address for unwinder
966 CFI_STARTPROC
967 movl %edx,%eax
968 push %edx
969 CFI_ADJUST_CFA_OFFSET 4
970 call *%ebx
971 push %eax
972 CFI_ADJUST_CFA_OFFSET 4
973 call do_exit
5f5db591 974 ud2 # padding for call trace
02ba1a32
AK		 975 CFI_ENDPROC
976ENDPROC(kernel_thread_helper)
977
5ead97c8 978#ifdef CONFIG_XEN
e2a81baf
JF		 979/* Xen doesn't set %esp to be precisely what the normal sysenter
980 entrypoint expects, so fix it up before using the normal path. */
981ENTRY(xen_sysenter_target)
982 RING0_INT_FRAME
983 addl $5*4, %esp /* remove xen-provided frame */
2ddf9b7b 984 CFI_ADJUST_CFA_OFFSET -5*4
e2a81baf 985 jmp sysenter_past_esp
557d7d4e 986 CFI_ENDPROC
e2a81baf 987
5ead97c8
JF		 988ENTRY(xen_hypervisor_callback)
989 CFI_STARTPROC
990 pushl $0
991 CFI_ADJUST_CFA_OFFSET 4
992 SAVE_ALL
993 TRACE_IRQS_OFF
9ec2b804
JF		 994
995 /* Check to see if we got the event in the critical
996 region in xen_iret_direct, after we've reenabled
997 events and checked for pending events. This simulates
998 iret instruction's behaviour where it delivers a
999 pending interrupt when enabling interrupts. */
1000 movl PT_EIP(%esp),%eax
1001 cmpl $xen_iret_start_crit,%eax
1002 jb 1f
1003 cmpl $xen_iret_end_crit,%eax
1004 jae 1f
1005
0f2c8769 1006 jmp xen_iret_crit_fixup
e2a81baf 1007
e2a81baf 1008ENTRY(xen_do_upcall)
b77797fb 10091: mov %esp, %eax
5ead97c8
JF		 1010 call xen_evtchn_do_upcall
1011 jmp ret_from_intr
1012 CFI_ENDPROC
1013ENDPROC(xen_hypervisor_callback)
1014
1015# Hypervisor uses this for application faults while it executes.
1016# We get here for two reasons:
1017# 1. Fault while reloading DS, ES, FS or GS
1018# 2. Fault while executing IRET
1019# Category 1 we fix up by reattempting the load, and zeroing the segment
1020# register if the load fails.
1021# Category 2 we fix up by jumping to do_iret_error. We cannot use the
1022# normal Linux return path in this case because if we use the IRET hypercall
1023# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1024# We distinguish between categories by maintaining a status value in EAX.
1025ENTRY(xen_failsafe_callback)
1026 CFI_STARTPROC
1027 pushl %eax
1028 CFI_ADJUST_CFA_OFFSET 4
1029 movl $1,%eax
10301: mov 4(%esp),%ds
10312: mov 8(%esp),%es
10323: mov 12(%esp),%fs
10334: mov 16(%esp),%gs
1034 testl %eax,%eax
1035 popl %eax
1036 CFI_ADJUST_CFA_OFFSET -4
1037 lea 16(%esp),%esp
1038 CFI_ADJUST_CFA_OFFSET -16
1039 jz 5f
1040 addl $16,%esp
1041 jmp iret_exc # EAX != 0 => Category 2 (Bad IRET)
10425: pushl $0 # EAX == 0 => Category 1 (Bad segment)
1043 CFI_ADJUST_CFA_OFFSET 4
1044 SAVE_ALL
1045 jmp ret_from_exception
1046 CFI_ENDPROC
1047
1048.section .fixup,"ax"
10496: xorl %eax,%eax
1050 movl %eax,4(%esp)
1051 jmp 1b
10527: xorl %eax,%eax
1053 movl %eax,8(%esp)
1054 jmp 2b
10558: xorl %eax,%eax
1056 movl %eax,12(%esp)
1057 jmp 3b
10589: xorl %eax,%eax
1059 movl %eax,16(%esp)
1060 jmp 4b
1061.previous
1062.section __ex_table,"a"
1063 .align 4
1064 .long 1b,6b
1065 .long 2b,7b
1066 .long 3b,8b
1067 .long 4b,9b
1068.previous
1069ENDPROC(xen_failsafe_callback)
1070
1071#endif /* CONFIG_XEN */
1072
606576ce 1073#ifdef CONFIG_FUNCTION_TRACER
d61f82d0
SR		 1074#ifdef CONFIG_DYNAMIC_FTRACE
1075
1076ENTRY(mcount)
d61f82d0
SR		 1077 ret
1078END(mcount)
1079
1080ENTRY(ftrace_caller)
60a7ecf4
SR		 1081 cmpl $0, function_trace_stop
1082 jne ftrace_stub
1083
d61f82d0
SR		 1084 pushl %eax
1085 pushl %ecx
1086 pushl %edx
1087 movl 0xc(%esp), %eax
1088 movl 0x4(%ebp), %edx
395a59d0 1089 subl $MCOUNT_INSN_SIZE, %eax
d61f82d0
SR		 1090
1091.globl ftrace_call
1092ftrace_call:
1093 call ftrace_stub
1094
1095 popl %edx
1096 popl %ecx
1097 popl %eax
5a45cfe1
SR		 1098#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1099.globl ftrace_graph_call
1100ftrace_graph_call:
1101 jmp ftrace_stub
1102#endif
d61f82d0
SR		 1103
1104.globl ftrace_stub
1105ftrace_stub:
1106 ret
1107END(ftrace_caller)
1108
1109#else /* ! CONFIG_DYNAMIC_FTRACE */
1110
16444a8a 1111ENTRY(mcount)
60a7ecf4
SR		 1112 cmpl $0, function_trace_stop
1113 jne ftrace_stub
1114
16444a8a
ACM		 1115 cmpl $ftrace_stub, ftrace_trace_function
1116 jnz trace
fb52607a 1117#ifdef CONFIG_FUNCTION_GRAPH_TRACER
c2324b69 1118 cmpl $ftrace_stub, ftrace_graph_return
fb52607a 1119 jnz ftrace_graph_caller
e49dc19c
SR		 1120
1121 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
1122 jnz ftrace_graph_caller
caf4b323 1123#endif
16444a8a
ACM		 1124.globl ftrace_stub
1125ftrace_stub:
1126 ret
1127
1128 /* taken from glibc */
1129trace:
1130 pushl %eax
1131 pushl %ecx
1132 pushl %edx
1133 movl 0xc(%esp), %eax
1134 movl 0x4(%ebp), %edx
395a59d0 1135 subl $MCOUNT_INSN_SIZE, %eax
16444a8a 1136
d61f82d0 1137 call *ftrace_trace_function
16444a8a
ACM		 1138
1139 popl %edx
1140 popl %ecx
1141 popl %eax
16444a8a
ACM		 1142 jmp ftrace_stub
1143END(mcount)
d61f82d0 1144#endif /* CONFIG_DYNAMIC_FTRACE */
606576ce 1145#endif /* CONFIG_FUNCTION_TRACER */
16444a8a 1146
fb52607a
FW		 1147#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1148ENTRY(ftrace_graph_caller)
e7d3737e
FW		 1149 cmpl $0, function_trace_stop
1150 jne ftrace_stub
1151
caf4b323
FW		 1152 pushl %eax
1153 pushl %ecx
1154 pushl %edx
1dc1c6ad 1155 movl 0xc(%esp), %edx
caf4b323 1156 lea 0x4(%ebp), %eax
bb4304c7 1157 subl $MCOUNT_INSN_SIZE, %edx
caf4b323 1158 call prepare_ftrace_return
caf4b323
FW		 1159 popl %edx
1160 popl %ecx
1161 popl %eax
e7d3737e 1162 ret
fb52607a 1163END(ftrace_graph_caller)
caf4b323
FW		 1164
1165.globl return_to_handler
1166return_to_handler:
1167 pushl $0
1168 pushl %eax
1169 pushl %ecx
1170 pushl %edx
1171 call ftrace_return_to_handler
1172 movl %eax, 0xc(%esp)
1173 popl %edx
1174 popl %ecx
1175 popl %eax
1176 ret
e7d3737e 1177#endif
16444a8a 1178
bb152f53 1179.section .rodata,"a"
541054d9 1180#include "syscall_table_32.S"
1da177e4
LT		 1181
1182syscall_table_size=(.-sys_call_table)
d211af05
AH		 1183
1184/*
1185 * Some functions should be protected against kprobes
1186 */
1187 .pushsection .kprobes.text, "ax"
1188
1189ENTRY(page_fault)
1190 RING0_EC_FRAME
1191 pushl $do_page_fault
1192 CFI_ADJUST_CFA_OFFSET 4
1193 ALIGN
1194error_code:
ccbeed3a
TH		 1195 /* the function address is in %gs's slot on the stack */
1196 pushl %fs
1197 CFI_ADJUST_CFA_OFFSET 4
1198 /*CFI_REL_OFFSET fs, 0*/
d211af05
AH		 1199 pushl %es
1200 CFI_ADJUST_CFA_OFFSET 4
1201 /*CFI_REL_OFFSET es, 0*/
1202 pushl %ds
1203 CFI_ADJUST_CFA_OFFSET 4
1204 /*CFI_REL_OFFSET ds, 0*/
1205 pushl %eax
1206 CFI_ADJUST_CFA_OFFSET 4
1207 CFI_REL_OFFSET eax, 0
1208 pushl %ebp
1209 CFI_ADJUST_CFA_OFFSET 4
1210 CFI_REL_OFFSET ebp, 0
1211 pushl %edi
1212 CFI_ADJUST_CFA_OFFSET 4
1213 CFI_REL_OFFSET edi, 0
1214 pushl %esi
1215 CFI_ADJUST_CFA_OFFSET 4
1216 CFI_REL_OFFSET esi, 0
1217 pushl %edx
1218 CFI_ADJUST_CFA_OFFSET 4
1219 CFI_REL_OFFSET edx, 0
1220 pushl %ecx
1221 CFI_ADJUST_CFA_OFFSET 4
1222 CFI_REL_OFFSET ecx, 0
1223 pushl %ebx
1224 CFI_ADJUST_CFA_OFFSET 4
1225 CFI_REL_OFFSET ebx, 0
1226 cld
d211af05
AH		 1227 movl $(__KERNEL_PERCPU), %ecx
1228 movl %ecx, %fs
1229 UNWIND_ESPFIX_STACK
ccbeed3a
TH		 1230 GS_TO_REG %ecx
1231 movl PT_GS(%esp), %edi # get the function address
d211af05
AH		 1232 movl PT_ORIG_EAX(%esp), %edx # get the error code
1233 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
ccbeed3a
TH		 1234 REG_TO_PTGS %ecx
1235 SET_KERNEL_GS %ecx
d211af05
AH		 1236 movl $(__USER_DS), %ecx
1237 movl %ecx, %ds
1238 movl %ecx, %es
1239 TRACE_IRQS_OFF
1240 movl %esp,%eax # pt_regs pointer
1241 call *%edi
1242 jmp ret_from_exception
1243 CFI_ENDPROC
1244END(page_fault)
1245
1246/*
1247 * Debug traps and NMI can happen at the one SYSENTER instruction
1248 * that sets up the real kernel stack. Check here, since we can't
1249 * allow the wrong stack to be used.
1250 *
1251 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
1252 * already pushed 3 words if it hits on the sysenter instruction:
1253 * eflags, cs and eip.
1254 *
1255 * We just load the right stack, and push the three (known) values
1256 * by hand onto the new stack - while updating the return eip past
1257 * the instruction that would have done it for sysenter.
1258 */
f0d96110
TH		 1259.macro FIX_STACK offset ok label
1260 cmpw $__KERNEL_CS, 4(%esp)
1261 jne \ok
1262\label:
1263 movl TSS_sysenter_sp0 + \offset(%esp), %esp
1264 CFI_DEF_CFA esp, 0
1265 CFI_UNDEFINED eip
1266 pushfl
1267 CFI_ADJUST_CFA_OFFSET 4
1268 pushl $__KERNEL_CS
1269 CFI_ADJUST_CFA_OFFSET 4
1270 pushl $sysenter_past_esp
1271 CFI_ADJUST_CFA_OFFSET 4
d211af05 1272 CFI_REL_OFFSET eip, 0
f0d96110 1273.endm
d211af05
AH		 1274
1275ENTRY(debug)
1276 RING0_INT_FRAME
1277 cmpl $ia32_sysenter_target,(%esp)
1278 jne debug_stack_correct
f0d96110 1279 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
d211af05
AH		 1280debug_stack_correct:
1281 pushl $-1 # mark this as an int
1282 CFI_ADJUST_CFA_OFFSET 4
1283 SAVE_ALL
1284 TRACE_IRQS_OFF
1285 xorl %edx,%edx # error code 0
1286 movl %esp,%eax # pt_regs pointer
1287 call do_debug
1288 jmp ret_from_exception
1289 CFI_ENDPROC
1290END(debug)
1291
1292/*
1293 * NMI is doubly nasty. It can happen _while_ we're handling
1294 * a debug fault, and the debug fault hasn't yet been able to
1295 * clear up the stack. So we first check whether we got an
1296 * NMI on the sysenter entry path, but after that we need to
1297 * check whether we got an NMI on the debug path where the debug
1298 * fault happened on the sysenter path.
1299 */
1300ENTRY(nmi)
1301 RING0_INT_FRAME
1302 pushl %eax
1303 CFI_ADJUST_CFA_OFFSET 4
1304 movl %ss, %eax
1305 cmpw $__ESPFIX_SS, %ax
1306 popl %eax
1307 CFI_ADJUST_CFA_OFFSET -4
1308 je nmi_espfix_stack
1309 cmpl $ia32_sysenter_target,(%esp)
1310 je nmi_stack_fixup
1311 pushl %eax
1312 CFI_ADJUST_CFA_OFFSET 4
1313 movl %esp,%eax
1314 /* Do not access memory above the end of our stack page,
1315 * it might not exist.
1316 */
1317 andl $(THREAD_SIZE-1),%eax
1318 cmpl $(THREAD_SIZE-20),%eax
1319 popl %eax
1320 CFI_ADJUST_CFA_OFFSET -4
1321 jae nmi_stack_correct
1322 cmpl $ia32_sysenter_target,12(%esp)
1323 je nmi_debug_stack_check
1324nmi_stack_correct:
1325 /* We have a RING0_INT_FRAME here */
1326 pushl %eax
1327 CFI_ADJUST_CFA_OFFSET 4
1328 SAVE_ALL
d211af05
AH		 1329 xorl %edx,%edx # zero error code
1330 movl %esp,%eax # pt_regs pointer
1331 call do_nmi
1332 jmp restore_nocheck_notrace
1333 CFI_ENDPROC
1334
1335nmi_stack_fixup:
1336 RING0_INT_FRAME
f0d96110 1337 FIX_STACK 12, nmi_stack_correct, 1
d211af05
AH		 1338 jmp nmi_stack_correct
1339
1340nmi_debug_stack_check:
1341 /* We have a RING0_INT_FRAME here */
1342 cmpw $__KERNEL_CS,16(%esp)
1343 jne nmi_stack_correct
1344 cmpl $debug,(%esp)
1345 jb nmi_stack_correct
1346 cmpl $debug_esp_fix_insn,(%esp)
1347 ja nmi_stack_correct
f0d96110 1348 FIX_STACK 24, nmi_stack_correct, 1
d211af05
AH		 1349 jmp nmi_stack_correct
1350
1351nmi_espfix_stack:
1352 /* We have a RING0_INT_FRAME here.
1353 *
1354 * create the pointer to lss back
1355 */
1356 pushl %ss
1357 CFI_ADJUST_CFA_OFFSET 4
1358 pushl %esp
1359 CFI_ADJUST_CFA_OFFSET 4
bda3a897 1360 addl $4, (%esp)
d211af05
AH		 1361 /* copy the iret frame of 12 bytes */
1362 .rept 3
1363 pushl 16(%esp)
1364 CFI_ADJUST_CFA_OFFSET 4
1365 .endr
1366 pushl %eax
1367 CFI_ADJUST_CFA_OFFSET 4
1368 SAVE_ALL
d211af05
AH		 1369 FIXUP_ESPFIX_STACK # %eax == %esp
1370 xorl %edx,%edx # zero error code
1371 call do_nmi
1372 RESTORE_REGS
1373 lss 12+4(%esp), %esp # back to espfix stack
1374 CFI_ADJUST_CFA_OFFSET -24
1375 jmp irq_return
1376 CFI_ENDPROC
1377END(nmi)
1378
1379ENTRY(int3)
1380 RING0_INT_FRAME
1381 pushl $-1 # mark this as an int
1382 CFI_ADJUST_CFA_OFFSET 4
1383 SAVE_ALL
1384 TRACE_IRQS_OFF
1385 xorl %edx,%edx # zero error code
1386 movl %esp,%eax # pt_regs pointer
1387 call do_int3
1388 jmp ret_from_exception
1389 CFI_ENDPROC
1390END(int3)
1391
1392ENTRY(general_protection)
1393 RING0_EC_FRAME
1394 pushl $do_general_protection
1395 CFI_ADJUST_CFA_OFFSET 4
1396 jmp error_code
1397 CFI_ENDPROC
1398END(general_protection)
1399
1400/*
1401 * End of kprobes section
1402 */
1403 .popsection
Linux 6.x block layer and io_uring tree(s)