projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
x86: use pt_regs pointer in do_device_not_available()
[linux-2.6-block.git] / arch / x86 / kernel / entry_32.S
CommitLineData
1da177e4 1/*
1da177e4
LT		 2 *
3 * Copyright (C) 1991, 1992 Linus Torvalds
4 */
5
6/*
7 * entry.S contains the system-call and fault low-level handling routines.
8 * This also contains the timer-interrupt handler, as well as all interrupts
9 * and faults that can result in a task-switch.
10 *
11 * NOTE: This code handles signal-recognition, which happens every time
12 * after a timer-interrupt and after each system call.
13 *
14 * I changed all the .align's to 4 (16 byte alignment), as that's faster
15 * on a 486.
16 *
889f21ce 17 * Stack layout in 'syscall_exit':
1da177e4
LT		 18 * ptrace needs to have all regs on the stack.
19 * if the order here is changed, it needs to be
20 * updated in fork.c:copy_process, signal.c:do_signal,
21 * ptrace.c and ptrace.h
22 *
23 * 0(%esp) - %ebx
24 * 4(%esp) - %ecx
25 * 8(%esp) - %edx
26 * C(%esp) - %esi
27 * 10(%esp) - %edi
28 * 14(%esp) - %ebp
29 * 18(%esp) - %eax
30 * 1C(%esp) - %ds
31 * 20(%esp) - %es
464d1a78 32 * 24(%esp) - %fs
ccbeed3a
TH		 33 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS
34 * 2C(%esp) - orig_eax
35 * 30(%esp) - %eip
36 * 34(%esp) - %cs
37 * 38(%esp) - %eflags
38 * 3C(%esp) - %oldesp
39 * 40(%esp) - %oldss
1da177e4
LT		 40 *
41 * "current" is in register %ebx during any slow entries.
42 */
43
1da177e4
LT		 44#include <linux/linkage.h>
45#include <asm/thread_info.h>
55f327fa 46#include <asm/irqflags.h>
1da177e4
LT		 47#include <asm/errno.h>
48#include <asm/segment.h>
49#include <asm/smp.h>
50#include <asm/page.h>
51#include <asm/desc.h>
be44d2aa 52#include <asm/percpu.h>
fe7cacc1 53#include <asm/dwarf2.h>
ab68ed98 54#include <asm/processor-flags.h>
395a59d0 55#include <asm/ftrace.h>
9b7dc567 56#include <asm/irq_vectors.h>
1da177e4 57
af0575bb
RM		 58/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
59#include <linux/elf-em.h>
60#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE)
61#define __AUDIT_ARCH_LE 0x40000000
62
63#ifndef CONFIG_AUDITSYSCALL
64#define sysenter_audit syscall_trace_entry
65#define sysexit_audit syscall_exit_work
66#endif
67
139ec7c4
RR		 68/*
69 * We use macros for low-level operations which need to be overridden
70 * for paravirtualization. The following will never clobber any registers:
71 * INTERRUPT_RETURN (aka. "iret")
72 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
d75cd22f 73 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
139ec7c4
RR		 74 *
75 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
76 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
77 * Allowing a register to be clobbered can shrink the paravirt replacement
78 * enough to patch inline, increasing performance.
79 */
80
1da177e4
LT		 81#define nr_syscalls ((syscall_table_size)/4)
82
1da177e4 83#ifdef CONFIG_PREEMPT
139ec7c4 84#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
1da177e4 85#else
139ec7c4 86#define preempt_stop(clobbers)
1da177e4
LT		 87#define resume_kernel restore_nocheck
88#endif
89
55f327fa
IM		 90.macro TRACE_IRQS_IRET
91#ifdef CONFIG_TRACE_IRQFLAGS
ab68ed98 92 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off?
55f327fa
IM		 93 jz 1f
94 TRACE_IRQS_ON
951:
96#endif
97.endm
98
4031ff38
AG		 99#ifdef CONFIG_VM86
100#define resume_userspace_sig check_userspace
101#else
102#define resume_userspace_sig resume_userspace
103#endif
104
ccbeed3a
TH		 105/*
106 * User gs save/restore
107 *
108 * %gs is used for userland TLS and kernel only uses it for stack
109 * canary which is required to be at %gs:20 by gcc. Read the comment
110 * at the top of stackprotector.h for more info.
111 *
112 * Local labels 98 and 99 are used.
113 */
114#ifdef CONFIG_X86_32_LAZY_GS
115
116 /* unfortunately push/pop can't be no-op */
117.macro PUSH_GS
118 pushl $0
119 CFI_ADJUST_CFA_OFFSET 4
120.endm
121.macro POP_GS pop=0
122 addl $(4 + \pop), %esp
123 CFI_ADJUST_CFA_OFFSET -(4 + \pop)
124.endm
125.macro POP_GS_EX
126.endm
127
128 /* all the rest are no-op */
129.macro PTGS_TO_GS
130.endm
131.macro PTGS_TO_GS_EX
132.endm
133.macro GS_TO_REG reg
134.endm
135.macro REG_TO_PTGS reg
136.endm
137.macro SET_KERNEL_GS reg
138.endm
139
140#else /* CONFIG_X86_32_LAZY_GS */
141
142.macro PUSH_GS
143 pushl %gs
144 CFI_ADJUST_CFA_OFFSET 4
145 /*CFI_REL_OFFSET gs, 0*/
146.endm
147
148.macro POP_GS pop=0
14998: popl %gs
150 CFI_ADJUST_CFA_OFFSET -4
151 /*CFI_RESTORE gs*/
152 .if \pop <> 0
153 add $\pop, %esp
154 CFI_ADJUST_CFA_OFFSET -\pop
155 .endif
156.endm
157.macro POP_GS_EX
158.pushsection .fixup, "ax"
15999: movl $0, (%esp)
160 jmp 98b
161.section __ex_table, "a"
162 .align 4
163 .long 98b, 99b
164.popsection
165.endm
166
167.macro PTGS_TO_GS
16898: mov PT_GS(%esp), %gs
169.endm
170.macro PTGS_TO_GS_EX
171.pushsection .fixup, "ax"
17299: movl $0, PT_GS(%esp)
173 jmp 98b
174.section __ex_table, "a"
175 .align 4
176 .long 98b, 99b
177.popsection
178.endm
179
180.macro GS_TO_REG reg
181 movl %gs, \reg
182 /*CFI_REGISTER gs, \reg*/
183.endm
184.macro REG_TO_PTGS reg
185 movl \reg, PT_GS(%esp)
186 /*CFI_REL_OFFSET gs, PT_GS*/
187.endm
188.macro SET_KERNEL_GS reg
60a5317f 189 movl $(__KERNEL_STACK_CANARY), \reg
ccbeed3a
TH		 190 movl \reg, %gs
191.endm
192
193#endif /* CONFIG_X86_32_LAZY_GS */
194
f0d96110
TH		 195.macro SAVE_ALL
196 cld
ccbeed3a 197 PUSH_GS
f0d96110
TH		 198 pushl %fs
199 CFI_ADJUST_CFA_OFFSET 4
200 /*CFI_REL_OFFSET fs, 0;*/
201 pushl %es
202 CFI_ADJUST_CFA_OFFSET 4
203 /*CFI_REL_OFFSET es, 0;*/
204 pushl %ds
205 CFI_ADJUST_CFA_OFFSET 4
206 /*CFI_REL_OFFSET ds, 0;*/
207 pushl %eax
208 CFI_ADJUST_CFA_OFFSET 4
209 CFI_REL_OFFSET eax, 0
210 pushl %ebp
211 CFI_ADJUST_CFA_OFFSET 4
212 CFI_REL_OFFSET ebp, 0
213 pushl %edi
214 CFI_ADJUST_CFA_OFFSET 4
215 CFI_REL_OFFSET edi, 0
216 pushl %esi
217 CFI_ADJUST_CFA_OFFSET 4
218 CFI_REL_OFFSET esi, 0
219 pushl %edx
220 CFI_ADJUST_CFA_OFFSET 4
221 CFI_REL_OFFSET edx, 0
222 pushl %ecx
223 CFI_ADJUST_CFA_OFFSET 4
224 CFI_REL_OFFSET ecx, 0
225 pushl %ebx
226 CFI_ADJUST_CFA_OFFSET 4
227 CFI_REL_OFFSET ebx, 0
228 movl $(__USER_DS), %edx
229 movl %edx, %ds
230 movl %edx, %es
231 movl $(__KERNEL_PERCPU), %edx
464d1a78 232 movl %edx, %fs
ccbeed3a 233 SET_KERNEL_GS %edx
f0d96110 234.endm
1da177e4 235
f0d96110
TH		 236.macro RESTORE_INT_REGS
237 popl %ebx
238 CFI_ADJUST_CFA_OFFSET -4
239 CFI_RESTORE ebx
240 popl %ecx
241 CFI_ADJUST_CFA_OFFSET -4
242 CFI_RESTORE ecx
243 popl %edx
244 CFI_ADJUST_CFA_OFFSET -4
245 CFI_RESTORE edx
246 popl %esi
247 CFI_ADJUST_CFA_OFFSET -4
248 CFI_RESTORE esi
249 popl %edi
250 CFI_ADJUST_CFA_OFFSET -4
251 CFI_RESTORE edi
252 popl %ebp
253 CFI_ADJUST_CFA_OFFSET -4
254 CFI_RESTORE ebp
255 popl %eax
256 CFI_ADJUST_CFA_OFFSET -4
fe7cacc1 257 CFI_RESTORE eax
f0d96110 258.endm
1da177e4 259
ccbeed3a 260.macro RESTORE_REGS pop=0
f0d96110
TH		 261 RESTORE_INT_REGS
2621: popl %ds
263 CFI_ADJUST_CFA_OFFSET -4
264 /*CFI_RESTORE ds;*/
2652: popl %es
266 CFI_ADJUST_CFA_OFFSET -4
267 /*CFI_RESTORE es;*/
2683: popl %fs
269 CFI_ADJUST_CFA_OFFSET -4
270 /*CFI_RESTORE fs;*/
ccbeed3a 271 POP_GS \pop
f0d96110
TH		 272.pushsection .fixup, "ax"
2734: movl $0, (%esp)
274 jmp 1b
2755: movl $0, (%esp)
276 jmp 2b
2776: movl $0, (%esp)
278 jmp 3b
279.section __ex_table, "a"
280 .align 4
281 .long 1b, 4b
282 .long 2b, 5b
283 .long 3b, 6b
f95d47ca 284.popsection
ccbeed3a 285 POP_GS_EX
f0d96110 286.endm
1da177e4 287
f0d96110
TH		 288.macro RING0_INT_FRAME
289 CFI_STARTPROC simple
290 CFI_SIGNAL_FRAME
291 CFI_DEF_CFA esp, 3*4
292 /*CFI_OFFSET cs, -2*4;*/
fe7cacc1 293 CFI_OFFSET eip, -3*4
f0d96110 294.endm
fe7cacc1 295
f0d96110
TH		 296.macro RING0_EC_FRAME
297 CFI_STARTPROC simple
298 CFI_SIGNAL_FRAME
299 CFI_DEF_CFA esp, 4*4
300 /*CFI_OFFSET cs, -2*4;*/
fe7cacc1 301 CFI_OFFSET eip, -3*4
f0d96110 302.endm
fe7cacc1 303
f0d96110
TH		 304.macro RING0_PTREGS_FRAME
305 CFI_STARTPROC simple
306 CFI_SIGNAL_FRAME
307 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX
308 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/
309 CFI_OFFSET eip, PT_EIP-PT_OLDESP
310 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/
311 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/
312 CFI_OFFSET eax, PT_EAX-PT_OLDESP
313 CFI_OFFSET ebp, PT_EBP-PT_OLDESP
314 CFI_OFFSET edi, PT_EDI-PT_OLDESP
315 CFI_OFFSET esi, PT_ESI-PT_OLDESP
316 CFI_OFFSET edx, PT_EDX-PT_OLDESP
317 CFI_OFFSET ecx, PT_ECX-PT_OLDESP
eb5b7b9d 318 CFI_OFFSET ebx, PT_EBX-PT_OLDESP
f0d96110 319.endm
1da177e4
LT		 320
321ENTRY(ret_from_fork)
fe7cacc1 322 CFI_STARTPROC
1da177e4 323 pushl %eax
25d7dfda 324 CFI_ADJUST_CFA_OFFSET 4
1da177e4
LT		 325 call schedule_tail
326 GET_THREAD_INFO(%ebp)
327 popl %eax
fe7cacc1 328 CFI_ADJUST_CFA_OFFSET -4
47a5c6fa
LT		 329 pushl $0x0202 # Reset kernel eflags
330 CFI_ADJUST_CFA_OFFSET 4
331 popfl
332 CFI_ADJUST_CFA_OFFSET -4
1da177e4 333 jmp syscall_exit
fe7cacc1 334 CFI_ENDPROC
47a55cd7 335END(ret_from_fork)
1da177e4
LT		 336
337/*
338 * Return to user mode is not as complex as all this looks,
339 * but we want the default path for a system call return to
340 * go as quickly as possible which is why some of this is
341 * less clear than it otherwise should be.
342 */
343
344 # userspace resumption stub bypassing syscall exit tracing
345 ALIGN
fe7cacc1 346 RING0_PTREGS_FRAME
1da177e4 347ret_from_exception:
139ec7c4 348 preempt_stop(CLBR_ANY)
1da177e4
LT		 349ret_from_intr:
350 GET_THREAD_INFO(%ebp)
4031ff38 351check_userspace:
eb5b7b9d
JF		 352 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
353 movb PT_CS(%esp), %al
ab68ed98 354 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
78be3706
RR		 355 cmpl $USER_RPL, %eax
356 jb resume_kernel # not returning to v8086 or userspace
f95d47ca 357
1da177e4 358ENTRY(resume_userspace)
c7e872e7 359 LOCKDEP_SYS_EXIT
139ec7c4 360 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
1da177e4
LT		 361 # setting need_resched or sigpending
362 # between sampling and the iret
e32e58a9 363 TRACE_IRQS_OFF
1da177e4
LT		 364 movl TI_flags(%ebp), %ecx
365 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
366 # int/exception return?
367 jne work_pending
368 jmp restore_all
47a55cd7 369END(ret_from_exception)
1da177e4
LT		 370
371#ifdef CONFIG_PREEMPT
372ENTRY(resume_kernel)
139ec7c4 373 DISABLE_INTERRUPTS(CLBR_ANY)
1da177e4
LT		 374 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
375 jnz restore_nocheck
376need_resched:
377 movl TI_flags(%ebp), %ecx # need_resched set ?
378 testb $_TIF_NEED_RESCHED, %cl
379 jz restore_all
ab68ed98 380 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ?
1da177e4
LT		 381 jz restore_all
382 call preempt_schedule_irq
383 jmp need_resched
47a55cd7 384END(resume_kernel)
1da177e4 385#endif
fe7cacc1 386 CFI_ENDPROC
1da177e4
LT		 387
388/* SYSENTER_RETURN points to after the "sysenter" instruction in
389 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
390
391 # sysenter call handler stub
0aa97fb2 392ENTRY(ia32_sysenter_target)
fe7cacc1 393 CFI_STARTPROC simple
adf14236 394 CFI_SIGNAL_FRAME
fe7cacc1
JB		 395 CFI_DEF_CFA esp, 0
396 CFI_REGISTER esp, ebp
faca6227 397 movl TSS_sysenter_sp0(%esp),%esp
1da177e4 398sysenter_past_esp:
55f327fa 399 /*
d93c870b
JF		 400 * Interrupts are disabled here, but we can't trace it until
401 * enough kernel state to call TRACE_IRQS_OFF can be called - but
402 * we immediately enable interrupts at that point anyway.
55f327fa 403 */
1da177e4 404 pushl $(__USER_DS)
fe7cacc1
JB		 405 CFI_ADJUST_CFA_OFFSET 4
406 /*CFI_REL_OFFSET ss, 0*/
1da177e4 407 pushl %ebp
fe7cacc1
JB		 408 CFI_ADJUST_CFA_OFFSET 4
409 CFI_REL_OFFSET esp, 0
1da177e4 410 pushfl
d93c870b 411 orl $X86_EFLAGS_IF, (%esp)
fe7cacc1 412 CFI_ADJUST_CFA_OFFSET 4
1da177e4 413 pushl $(__USER_CS)
fe7cacc1
JB		 414 CFI_ADJUST_CFA_OFFSET 4
415 /*CFI_REL_OFFSET cs, 0*/
e6e5494c
IM		 416 /*
417 * Push current_thread_info()->sysenter_return to the stack.
418 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
419 * pushed above; +8 corresponds to copy_thread's esp0 setting.
420 */
421 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
fe7cacc1
JB		 422 CFI_ADJUST_CFA_OFFSET 4
423 CFI_REL_OFFSET eip, 0
1da177e4 424
d93c870b
JF		 425 pushl %eax
426 CFI_ADJUST_CFA_OFFSET 4
427 SAVE_ALL
428 ENABLE_INTERRUPTS(CLBR_NONE)
429
1da177e4
LT		 430/*
431 * Load the potential sixth argument from user stack.
432 * Careful about security.
433 */
434 cmpl $__PAGE_OFFSET-3,%ebp
435 jae syscall_fault
4361: movl (%ebp),%ebp
d93c870b 437 movl %ebp,PT_EBP(%esp)
1da177e4
LT		 438.section __ex_table,"a"
439 .align 4
440 .long 1b,syscall_fault
441.previous
442
1da177e4
LT		 443 GET_THREAD_INFO(%ebp)
444
445 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
d4d67150 446 testw $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
af0575bb
RM		 447 jnz sysenter_audit
448sysenter_do_call:
1da177e4
LT		 449 cmpl $(nr_syscalls), %eax
450 jae syscall_badsys
451 call *sys_call_table(,%eax,4)
eb5b7b9d 452 movl %eax,PT_EAX(%esp)
c7e872e7 453 LOCKDEP_SYS_EXIT
42c24fa2 454 DISABLE_INTERRUPTS(CLBR_ANY)
55f327fa 455 TRACE_IRQS_OFF
1da177e4
LT		 456 movl TI_flags(%ebp), %ecx
457 testw $_TIF_ALLWORK_MASK, %cx
af0575bb
RM		 458 jne sysexit_audit
459sysenter_exit:
1da177e4 460/* if something modifies registers it must also disable sysexit */
eb5b7b9d
JF		 461 movl PT_EIP(%esp), %edx
462 movl PT_OLDESP(%esp), %ecx
1da177e4 463 xorl %ebp,%ebp
55f327fa 464 TRACE_IRQS_ON
464d1a78 4651: mov PT_FS(%esp), %fs
ccbeed3a 466 PTGS_TO_GS
d75cd22f 467 ENABLE_INTERRUPTS_SYSEXIT
af0575bb
RM		 468
469#ifdef CONFIG_AUDITSYSCALL
470sysenter_audit:
471 testw $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
472 jnz syscall_trace_entry
473 addl $4,%esp
474 CFI_ADJUST_CFA_OFFSET -4
475 /* %esi already in 8(%esp) 6th arg: 4th syscall arg */
476 /* %edx already in 4(%esp) 5th arg: 3rd syscall arg */
477 /* %ecx already in 0(%esp) 4th arg: 2nd syscall arg */
478 movl %ebx,%ecx /* 3rd arg: 1st syscall arg */
479 movl %eax,%edx /* 2nd arg: syscall number */
480 movl $AUDIT_ARCH_I386,%eax /* 1st arg: audit arch */
481 call audit_syscall_entry
482 pushl %ebx
483 CFI_ADJUST_CFA_OFFSET 4
484 movl PT_EAX(%esp),%eax /* reload syscall number */
485 jmp sysenter_do_call
486
487sysexit_audit:
488 testw $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %cx
489 jne syscall_exit_work
490 TRACE_IRQS_ON
491 ENABLE_INTERRUPTS(CLBR_ANY)
492 movl %eax,%edx /* second arg, syscall return value */
493 cmpl $0,%eax /* is it < 0? */
494 setl %al /* 1 if so, 0 if not */
495 movzbl %al,%eax /* zero-extend that */
496 inc %eax /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
497 call audit_syscall_exit
498 DISABLE_INTERRUPTS(CLBR_ANY)
499 TRACE_IRQS_OFF
500 movl TI_flags(%ebp), %ecx
501 testw $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %cx
502 jne syscall_exit_work
503 movl PT_EAX(%esp),%eax /* reload syscall return value */
504 jmp sysenter_exit
505#endif
506
fe7cacc1 507 CFI_ENDPROC
f95d47ca 508.pushsection .fixup,"ax"
464d1a78 5092: movl $0,PT_FS(%esp)
f95d47ca
JF		 510 jmp 1b
511.section __ex_table,"a"
512 .align 4
513 .long 1b,2b
514.popsection
ccbeed3a 515 PTGS_TO_GS_EX
0aa97fb2 516ENDPROC(ia32_sysenter_target)
1da177e4
LT		 517
518 # system call handler stub
519ENTRY(system_call)
fe7cacc1 520 RING0_INT_FRAME # can't unwind into user space anyway
1da177e4 521 pushl %eax # save orig_eax
fe7cacc1 522 CFI_ADJUST_CFA_OFFSET 4
1da177e4
LT		 523 SAVE_ALL
524 GET_THREAD_INFO(%ebp)
ed75e8d5 525 # system call tracing in operation / emulation
1da177e4 526 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
d4d67150 527 testw $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
1da177e4
LT		 528 jnz syscall_trace_entry
529 cmpl $(nr_syscalls), %eax
530 jae syscall_badsys
531syscall_call:
532 call *sys_call_table(,%eax,4)
eb5b7b9d 533 movl %eax,PT_EAX(%esp) # store the return value
1da177e4 534syscall_exit:
c7e872e7 535 LOCKDEP_SYS_EXIT
139ec7c4 536 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
1da177e4
LT		 537 # setting need_resched or sigpending
538 # between sampling and the iret
55f327fa 539 TRACE_IRQS_OFF
1da177e4
LT		 540 movl TI_flags(%ebp), %ecx
541 testw $_TIF_ALLWORK_MASK, %cx # current->work
542 jne syscall_exit_work
543
544restore_all:
eb5b7b9d
JF		 545 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
546 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
5df24082
SS		 547 # are returning to the kernel.
548 # See comments in process.c:copy_thread() for details.
eb5b7b9d
JF		 549 movb PT_OLDSS(%esp), %ah
550 movb PT_CS(%esp), %al
ab68ed98 551 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
78be3706 552 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
fe7cacc1 553 CFI_REMEMBER_STATE
1da177e4
LT		 554 je ldt_ss # returning to user-space with LDT SS
555restore_nocheck:
55f327fa
IM		 556 TRACE_IRQS_IRET
557restore_nocheck_notrace:
ccbeed3a 558 RESTORE_REGS 4 # skip orig_eax/error_code
fe7cacc1 559 CFI_ADJUST_CFA_OFFSET -4
f7f3d791 560irq_return:
3701d863 561 INTERRUPT_RETURN
1da177e4 562.section .fixup,"ax"
90e9f536 563ENTRY(iret_exc)
a879cbbb
LT		 564 pushl $0 # no error code
565 pushl $do_iret_error
566 jmp error_code
1da177e4
LT		 567.previous
568.section __ex_table,"a"
569 .align 4
3701d863 570 .long irq_return,iret_exc
1da177e4
LT		 571.previous
572
fe7cacc1 573 CFI_RESTORE_STATE
1da177e4 574ldt_ss:
eb5b7b9d 575 larl PT_OLDSS(%esp), %eax
1da177e4
LT		 576 jnz restore_nocheck
577 testl $0x00400000, %eax # returning to 32bit stack?
578 jnz restore_nocheck # allright, normal return
d3561b7f
RR		 579
580#ifdef CONFIG_PARAVIRT
581 /*
582 * The kernel can't run on a non-flat stack if paravirt mode
583 * is active. Rather than try to fixup the high bits of
584 * ESP, bypass this code entirely. This may break DOSemu
585 * and/or Wine support in a paravirt VM, although the option
586 * is still available to implement the setting of the high
587 * 16-bits in the INTERRUPT_RETURN paravirt-op.
588 */
93b1eab3 589 cmpl $0, pv_info+PARAVIRT_enabled
d3561b7f
RR		 590 jne restore_nocheck
591#endif
592
1da177e4
LT		 593 /* If returning to userspace with 16bit stack,
594 * try to fix the higher word of ESP, as the CPU
595 * won't restore it.
596 * This is an "official" bug of all the x86-compatible
597 * CPUs, which we can try to work around to make
598 * dosemu and wine happy. */
eb5b7b9d 599 movl PT_OLDESP(%esp), %eax
be44d2aa
SS		 600 movl %esp, %edx
601 call patch_espfix_desc
602 pushl $__ESPFIX_SS
603 CFI_ADJUST_CFA_OFFSET 4
604 pushl %eax
605 CFI_ADJUST_CFA_OFFSET 4
139ec7c4 606 DISABLE_INTERRUPTS(CLBR_EAX)
55f327fa 607 TRACE_IRQS_OFF
be44d2aa
SS		 608 lss (%esp), %esp
609 CFI_ADJUST_CFA_OFFSET -8
610 jmp restore_nocheck
fe7cacc1 611 CFI_ENDPROC
47a55cd7 612ENDPROC(system_call)
1da177e4
LT		 613
614 # perform work that needs to be done immediately before resumption
615 ALIGN
fe7cacc1 616 RING0_PTREGS_FRAME # can't unwind into user space anyway
1da177e4
LT		 617work_pending:
618 testb $_TIF_NEED_RESCHED, %cl
619 jz work_notifysig
620work_resched:
621 call schedule
c7e872e7 622 LOCKDEP_SYS_EXIT
139ec7c4 623 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
1da177e4
LT		 624 # setting need_resched or sigpending
625 # between sampling and the iret
55f327fa 626 TRACE_IRQS_OFF
1da177e4
LT		 627 movl TI_flags(%ebp), %ecx
628 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
629 # than syscall tracing?
630 jz restore_all
631 testb $_TIF_NEED_RESCHED, %cl
632 jnz work_resched
633
634work_notifysig: # deal with pending signals and
635 # notify-resume requests
74b47a78 636#ifdef CONFIG_VM86
ab68ed98 637 testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
1da177e4
LT		 638 movl %esp, %eax
639 jne work_notifysig_v86 # returning to kernel-space or
640 # vm86-space
641 xorl %edx, %edx
642 call do_notify_resume
4031ff38 643 jmp resume_userspace_sig
1da177e4
LT		 644
645 ALIGN
646work_notifysig_v86:
647 pushl %ecx # save ti_flags for do_notify_resume
fe7cacc1 648 CFI_ADJUST_CFA_OFFSET 4
1da177e4
LT		 649 call save_v86_state # %eax contains pt_regs pointer
650 popl %ecx
fe7cacc1 651 CFI_ADJUST_CFA_OFFSET -4
1da177e4 652 movl %eax, %esp
74b47a78
JK		 653#else
654 movl %esp, %eax
655#endif
1da177e4
LT		 656 xorl %edx, %edx
657 call do_notify_resume
4031ff38 658 jmp resume_userspace_sig
47a55cd7 659END(work_pending)
1da177e4
LT		 660
661 # perform syscall exit tracing
662 ALIGN
663syscall_trace_entry:
eb5b7b9d 664 movl $-ENOSYS,PT_EAX(%esp)
1da177e4 665 movl %esp, %eax
d4d67150
RM		 666 call syscall_trace_enter
667 /* What it returned is what we'll actually use. */
1da177e4
LT		 668 cmpl $(nr_syscalls), %eax
669 jnae syscall_call
670 jmp syscall_exit
47a55cd7 671END(syscall_trace_entry)
1da177e4
LT		 672
673 # perform syscall exit tracing
674 ALIGN
675syscall_exit_work:
d4d67150 676 testb $_TIF_WORK_SYSCALL_EXIT, %cl
1da177e4 677 jz work_pending
55f327fa 678 TRACE_IRQS_ON
d4d67150 679 ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call
1da177e4
LT		 680 # schedule() instead
681 movl %esp, %eax
d4d67150 682 call syscall_trace_leave
1da177e4 683 jmp resume_userspace
47a55cd7 684END(syscall_exit_work)
fe7cacc1 685 CFI_ENDPROC
1da177e4 686
fe7cacc1 687 RING0_INT_FRAME # can't unwind into user space anyway
1da177e4 688syscall_fault:
1da177e4 689 GET_THREAD_INFO(%ebp)
eb5b7b9d 690 movl $-EFAULT,PT_EAX(%esp)
1da177e4 691 jmp resume_userspace
47a55cd7 692END(syscall_fault)
1da177e4 693
1da177e4 694syscall_badsys:
eb5b7b9d 695 movl $-ENOSYS,PT_EAX(%esp)
1da177e4 696 jmp resume_userspace
47a55cd7 697END(syscall_badsys)
fe7cacc1 698 CFI_ENDPROC
1da177e4 699
f0d96110
TH		 700.macro FIXUP_ESPFIX_STACK
701 /* since we are on a wrong stack, we cant make it a C code :( */
702 PER_CPU(gdt_page, %ebx)
703 GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah)
704 addl %esp, %eax
705 pushl $__KERNEL_DS
706 CFI_ADJUST_CFA_OFFSET 4
707 pushl %eax
708 CFI_ADJUST_CFA_OFFSET 4
709 lss (%esp), %esp
710 CFI_ADJUST_CFA_OFFSET -8
711.endm
712.macro UNWIND_ESPFIX_STACK
713 movl %ss, %eax
714 /* see if on espfix stack */
715 cmpw $__ESPFIX_SS, %ax
716 jne 27f
717 movl $__KERNEL_DS, %eax
718 movl %eax, %ds
719 movl %eax, %es
720 /* switch to normal stack */
721 FIXUP_ESPFIX_STACK
72227:
723.endm
1da177e4
LT		 724
725/*
b7c6244f
PA		 726 * Build the entry stubs and pointer table with some assembler magic.
727 * We pack 7 stubs into a single 32-byte chunk, which will fit in a
728 * single cache line on all modern x86 implementations.
1da177e4 729 */
4687518c 730.section .init.rodata,"a"
1da177e4
LT		 731ENTRY(interrupt)
732.text
b7c6244f
PA		 733 .p2align 5
734 .p2align CONFIG_X86_L1_CACHE_SHIFT
1da177e4 735ENTRY(irq_entries_start)
fe7cacc1 736 RING0_INT_FRAME
4687518c 737vector=FIRST_EXTERNAL_VECTOR
b7c6244f
PA		 738.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
739 .balign 32
740 .rept 7
741 .if vector < NR_VECTORS
8665596e 742 .if vector <> FIRST_EXTERNAL_VECTOR
fe7cacc1 743 CFI_ADJUST_CFA_OFFSET -4
b7c6244f
PA		 744 .endif
7451: pushl $(~vector+0x80) /* Note: always in signed byte range */
fe7cacc1 746 CFI_ADJUST_CFA_OFFSET 4
8665596e 747 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
b7c6244f
PA		 748 jmp 2f
749 .endif
750 .previous
1da177e4 751 .long 1b
b7c6244f 752 .text
1da177e4 753vector=vector+1
b7c6244f
PA		 754 .endif
755 .endr
7562: jmp common_interrupt
1da177e4 757.endr
47a55cd7
JB		 758END(irq_entries_start)
759
760.previous
761END(interrupt)
762.previous
1da177e4 763
55f327fa
IM		 764/*
765 * the CPU automatically disables interrupts when executing an IRQ vector,
766 * so IRQ-flags tracing has to follow that:
767 */
b7c6244f 768 .p2align CONFIG_X86_L1_CACHE_SHIFT
1da177e4 769common_interrupt:
b7c6244f 770 addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */
1da177e4 771 SAVE_ALL
55f327fa 772 TRACE_IRQS_OFF
1da177e4
LT		 773 movl %esp,%eax
774 call do_IRQ
775 jmp ret_from_intr
47a55cd7 776ENDPROC(common_interrupt)
fe7cacc1 777 CFI_ENDPROC
1da177e4 778
02cf94c3 779#define BUILD_INTERRUPT3(name, nr, fn) \
1da177e4 780ENTRY(name) \
fe7cacc1 781 RING0_INT_FRAME; \
19eadf98 782 pushl $~(nr); \
fe7cacc1
JB		 783 CFI_ADJUST_CFA_OFFSET 4; \
784 SAVE_ALL; \
55f327fa 785 TRACE_IRQS_OFF \
1da177e4 786 movl %esp,%eax; \
02cf94c3 787 call fn; \
55f327fa 788 jmp ret_from_intr; \
47a55cd7
JB		 789 CFI_ENDPROC; \
790ENDPROC(name)
1da177e4 791
02cf94c3
TH		 792#define BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(name, nr, smp_##name)
793
1da177e4
LT		 794/* The include is where all of the SMP etc. interrupts come from */
795#include "entry_arch.h"
796
1da177e4 797ENTRY(coprocessor_error)
fe7cacc1 798 RING0_INT_FRAME
1da177e4 799 pushl $0
fe7cacc1 800 CFI_ADJUST_CFA_OFFSET 4
1da177e4 801 pushl $do_coprocessor_error
fe7cacc1 802 CFI_ADJUST_CFA_OFFSET 4
1da177e4 803 jmp error_code
fe7cacc1 804 CFI_ENDPROC
47a55cd7 805END(coprocessor_error)
1da177e4
LT		 806
807ENTRY(simd_coprocessor_error)
fe7cacc1 808 RING0_INT_FRAME
1da177e4 809 pushl $0
fe7cacc1 810 CFI_ADJUST_CFA_OFFSET 4
1da177e4 811 pushl $do_simd_coprocessor_error
fe7cacc1 812 CFI_ADJUST_CFA_OFFSET 4
1da177e4 813 jmp error_code
fe7cacc1 814 CFI_ENDPROC
47a55cd7 815END(simd_coprocessor_error)
1da177e4
LT		 816
817ENTRY(device_not_available)
fe7cacc1 818 RING0_INT_FRAME
1da177e4 819 pushl $-1 # mark this as an int
fe7cacc1 820 CFI_ADJUST_CFA_OFFSET 4
7643e9b9 821 pushl $do_device_not_available
fe7cacc1 822 CFI_ADJUST_CFA_OFFSET 4
7643e9b9 823 jmp error_code
fe7cacc1 824 CFI_ENDPROC
47a55cd7 825END(device_not_available)
1da177e4 826
d3561b7f
RR		 827#ifdef CONFIG_PARAVIRT
828ENTRY(native_iret)
3701d863 829 iret
d3561b7f
RR		 830.section __ex_table,"a"
831 .align 4
3701d863 832 .long native_iret, iret_exc
d3561b7f 833.previous
47a55cd7 834END(native_iret)
d3561b7f 835
d75cd22f 836ENTRY(native_irq_enable_sysexit)
d3561b7f
RR		 837 sti
838 sysexit
d75cd22f 839END(native_irq_enable_sysexit)
d3561b7f
RR		 840#endif
841
1da177e4 842ENTRY(overflow)
fe7cacc1 843 RING0_INT_FRAME
1da177e4 844 pushl $0
fe7cacc1 845 CFI_ADJUST_CFA_OFFSET 4
1da177e4 846 pushl $do_overflow
fe7cacc1 847 CFI_ADJUST_CFA_OFFSET 4
1da177e4 848 jmp error_code
fe7cacc1 849 CFI_ENDPROC
47a55cd7 850END(overflow)
1da177e4
LT		 851
852ENTRY(bounds)
fe7cacc1 853 RING0_INT_FRAME
1da177e4 854 pushl $0
fe7cacc1 855 CFI_ADJUST_CFA_OFFSET 4
1da177e4 856 pushl $do_bounds
fe7cacc1 857 CFI_ADJUST_CFA_OFFSET 4
1da177e4 858 jmp error_code
fe7cacc1 859 CFI_ENDPROC
47a55cd7 860END(bounds)
1da177e4
LT		 861
862ENTRY(invalid_op)
fe7cacc1 863 RING0_INT_FRAME
1da177e4 864 pushl $0
fe7cacc1 865 CFI_ADJUST_CFA_OFFSET 4
1da177e4 866 pushl $do_invalid_op
fe7cacc1 867 CFI_ADJUST_CFA_OFFSET 4
1da177e4 868 jmp error_code
fe7cacc1 869 CFI_ENDPROC
47a55cd7 870END(invalid_op)
1da177e4
LT		 871
872ENTRY(coprocessor_segment_overrun)
fe7cacc1 873 RING0_INT_FRAME
1da177e4 874 pushl $0
fe7cacc1 875 CFI_ADJUST_CFA_OFFSET 4
1da177e4 876 pushl $do_coprocessor_segment_overrun
fe7cacc1 877 CFI_ADJUST_CFA_OFFSET 4
1da177e4 878 jmp error_code
fe7cacc1 879 CFI_ENDPROC
47a55cd7 880END(coprocessor_segment_overrun)
1da177e4
LT		 881
882ENTRY(invalid_TSS)
fe7cacc1 883 RING0_EC_FRAME
1da177e4 884 pushl $do_invalid_TSS
fe7cacc1 885 CFI_ADJUST_CFA_OFFSET 4
1da177e4 886 jmp error_code
fe7cacc1 887 CFI_ENDPROC
47a55cd7 888END(invalid_TSS)
1da177e4
LT		 889
890ENTRY(segment_not_present)
fe7cacc1 891 RING0_EC_FRAME
1da177e4 892 pushl $do_segment_not_present
fe7cacc1 893 CFI_ADJUST_CFA_OFFSET 4
1da177e4 894 jmp error_code
fe7cacc1 895 CFI_ENDPROC
47a55cd7 896END(segment_not_present)
1da177e4
LT		 897
898ENTRY(stack_segment)
fe7cacc1 899 RING0_EC_FRAME
1da177e4 900 pushl $do_stack_segment
fe7cacc1 901 CFI_ADJUST_CFA_OFFSET 4
1da177e4 902 jmp error_code
fe7cacc1 903 CFI_ENDPROC
47a55cd7 904END(stack_segment)
1da177e4 905
1da177e4 906ENTRY(alignment_check)
fe7cacc1 907 RING0_EC_FRAME
1da177e4 908 pushl $do_alignment_check
fe7cacc1 909 CFI_ADJUST_CFA_OFFSET 4
1da177e4 910 jmp error_code
fe7cacc1 911 CFI_ENDPROC
47a55cd7 912END(alignment_check)
1da177e4 913
d28c4393
P		 914ENTRY(divide_error)
915 RING0_INT_FRAME
916 pushl $0 # no error code
917 CFI_ADJUST_CFA_OFFSET 4
918 pushl $do_divide_error
fe7cacc1 919 CFI_ADJUST_CFA_OFFSET 4
1da177e4 920 jmp error_code
fe7cacc1 921 CFI_ENDPROC
47a55cd7 922END(divide_error)
1da177e4
LT		 923
924#ifdef CONFIG_X86_MCE
925ENTRY(machine_check)
fe7cacc1 926 RING0_INT_FRAME
1da177e4 927 pushl $0
fe7cacc1 928 CFI_ADJUST_CFA_OFFSET 4
d2f6f7ae 929 pushl machine_check_vector
fe7cacc1 930 CFI_ADJUST_CFA_OFFSET 4
1da177e4 931 jmp error_code
fe7cacc1 932 CFI_ENDPROC
47a55cd7 933END(machine_check)
1da177e4
LT		 934#endif
935
936ENTRY(spurious_interrupt_bug)
fe7cacc1 937 RING0_INT_FRAME
1da177e4 938 pushl $0
fe7cacc1 939 CFI_ADJUST_CFA_OFFSET 4
1da177e4 940 pushl $do_spurious_interrupt_bug
fe7cacc1 941 CFI_ADJUST_CFA_OFFSET 4
1da177e4 942 jmp error_code
fe7cacc1 943 CFI_ENDPROC
47a55cd7 944END(spurious_interrupt_bug)
1da177e4 945
02ba1a32
AK		 946ENTRY(kernel_thread_helper)
947 pushl $0 # fake return address for unwinder
948 CFI_STARTPROC
949 movl %edx,%eax
950 push %edx
951 CFI_ADJUST_CFA_OFFSET 4
952 call *%ebx
953 push %eax
954 CFI_ADJUST_CFA_OFFSET 4
955 call do_exit
5f5db591 956 ud2 # padding for call trace
02ba1a32
AK		 957 CFI_ENDPROC
958ENDPROC(kernel_thread_helper)
959
5ead97c8 960#ifdef CONFIG_XEN
e2a81baf
JF		 961/* Xen doesn't set %esp to be precisely what the normal sysenter
962 entrypoint expects, so fix it up before using the normal path. */
963ENTRY(xen_sysenter_target)
964 RING0_INT_FRAME
965 addl $5*4, %esp /* remove xen-provided frame */
2ddf9b7b 966 CFI_ADJUST_CFA_OFFSET -5*4
e2a81baf 967 jmp sysenter_past_esp
557d7d4e 968 CFI_ENDPROC
e2a81baf 969
5ead97c8
JF		 970ENTRY(xen_hypervisor_callback)
971 CFI_STARTPROC
972 pushl $0
973 CFI_ADJUST_CFA_OFFSET 4
974 SAVE_ALL
975 TRACE_IRQS_OFF
9ec2b804
JF		 976
977 /* Check to see if we got the event in the critical
978 region in xen_iret_direct, after we've reenabled
979 events and checked for pending events. This simulates
980 iret instruction's behaviour where it delivers a
981 pending interrupt when enabling interrupts. */
982 movl PT_EIP(%esp),%eax
983 cmpl $xen_iret_start_crit,%eax
984 jb 1f
985 cmpl $xen_iret_end_crit,%eax
986 jae 1f
987
0f2c8769 988 jmp xen_iret_crit_fixup
e2a81baf 989
e2a81baf 990ENTRY(xen_do_upcall)
b77797fb 9911: mov %esp, %eax
5ead97c8
JF		 992 call xen_evtchn_do_upcall
993 jmp ret_from_intr
994 CFI_ENDPROC
995ENDPROC(xen_hypervisor_callback)
996
997# Hypervisor uses this for application faults while it executes.
998# We get here for two reasons:
999# 1. Fault while reloading DS, ES, FS or GS
1000# 2. Fault while executing IRET
1001# Category 1 we fix up by reattempting the load, and zeroing the segment
1002# register if the load fails.
1003# Category 2 we fix up by jumping to do_iret_error. We cannot use the
1004# normal Linux return path in this case because if we use the IRET hypercall
1005# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1006# We distinguish between categories by maintaining a status value in EAX.
1007ENTRY(xen_failsafe_callback)
1008 CFI_STARTPROC
1009 pushl %eax
1010 CFI_ADJUST_CFA_OFFSET 4
1011 movl $1,%eax
10121: mov 4(%esp),%ds
10132: mov 8(%esp),%es
10143: mov 12(%esp),%fs
10154: mov 16(%esp),%gs
1016 testl %eax,%eax
1017 popl %eax
1018 CFI_ADJUST_CFA_OFFSET -4
1019 lea 16(%esp),%esp
1020 CFI_ADJUST_CFA_OFFSET -16
1021 jz 5f
1022 addl $16,%esp
1023 jmp iret_exc # EAX != 0 => Category 2 (Bad IRET)
10245: pushl $0 # EAX == 0 => Category 1 (Bad segment)
1025 CFI_ADJUST_CFA_OFFSET 4
1026 SAVE_ALL
1027 jmp ret_from_exception
1028 CFI_ENDPROC
1029
1030.section .fixup,"ax"
10316: xorl %eax,%eax
1032 movl %eax,4(%esp)
1033 jmp 1b
10347: xorl %eax,%eax
1035 movl %eax,8(%esp)
1036 jmp 2b
10378: xorl %eax,%eax
1038 movl %eax,12(%esp)
1039 jmp 3b
10409: xorl %eax,%eax
1041 movl %eax,16(%esp)
1042 jmp 4b
1043.previous
1044.section __ex_table,"a"
1045 .align 4
1046 .long 1b,6b
1047 .long 2b,7b
1048 .long 3b,8b
1049 .long 4b,9b
1050.previous
1051ENDPROC(xen_failsafe_callback)
1052
1053#endif /* CONFIG_XEN */
1054
606576ce 1055#ifdef CONFIG_FUNCTION_TRACER
d61f82d0
SR		 1056#ifdef CONFIG_DYNAMIC_FTRACE
1057
1058ENTRY(mcount)
d61f82d0
SR		 1059 ret
1060END(mcount)
1061
1062ENTRY(ftrace_caller)
60a7ecf4
SR		 1063 cmpl $0, function_trace_stop
1064 jne ftrace_stub
1065
d61f82d0
SR		 1066 pushl %eax
1067 pushl %ecx
1068 pushl %edx
1069 movl 0xc(%esp), %eax
1070 movl 0x4(%ebp), %edx
395a59d0 1071 subl $MCOUNT_INSN_SIZE, %eax
d61f82d0
SR		 1072
1073.globl ftrace_call
1074ftrace_call:
1075 call ftrace_stub
1076
1077 popl %edx
1078 popl %ecx
1079 popl %eax
5a45cfe1
SR		 1080#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1081.globl ftrace_graph_call
1082ftrace_graph_call:
1083 jmp ftrace_stub
1084#endif
d61f82d0
SR		 1085
1086.globl ftrace_stub
1087ftrace_stub:
1088 ret
1089END(ftrace_caller)
1090
1091#else /* ! CONFIG_DYNAMIC_FTRACE */
1092
16444a8a 1093ENTRY(mcount)
60a7ecf4
SR		 1094 cmpl $0, function_trace_stop
1095 jne ftrace_stub
1096
16444a8a
ACM		 1097 cmpl $ftrace_stub, ftrace_trace_function
1098 jnz trace
fb52607a 1099#ifdef CONFIG_FUNCTION_GRAPH_TRACER
c2324b69 1100 cmpl $ftrace_stub, ftrace_graph_return
fb52607a 1101 jnz ftrace_graph_caller
e49dc19c
SR		 1102
1103 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
1104 jnz ftrace_graph_caller
caf4b323 1105#endif
16444a8a
ACM		 1106.globl ftrace_stub
1107ftrace_stub:
1108 ret
1109
1110 /* taken from glibc */
1111trace:
1112 pushl %eax
1113 pushl %ecx
1114 pushl %edx
1115 movl 0xc(%esp), %eax
1116 movl 0x4(%ebp), %edx
395a59d0 1117 subl $MCOUNT_INSN_SIZE, %eax
16444a8a 1118
d61f82d0 1119 call *ftrace_trace_function
16444a8a
ACM		 1120
1121 popl %edx
1122 popl %ecx
1123 popl %eax
16444a8a
ACM		 1124 jmp ftrace_stub
1125END(mcount)
d61f82d0 1126#endif /* CONFIG_DYNAMIC_FTRACE */
606576ce 1127#endif /* CONFIG_FUNCTION_TRACER */
16444a8a 1128
fb52607a
FW		 1129#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1130ENTRY(ftrace_graph_caller)
e7d3737e
FW		 1131 cmpl $0, function_trace_stop
1132 jne ftrace_stub
1133
caf4b323
FW		 1134 pushl %eax
1135 pushl %ecx
1136 pushl %edx
1dc1c6ad 1137 movl 0xc(%esp), %edx
caf4b323 1138 lea 0x4(%ebp), %eax
bb4304c7 1139 subl $MCOUNT_INSN_SIZE, %edx
caf4b323 1140 call prepare_ftrace_return
caf4b323
FW		 1141 popl %edx
1142 popl %ecx
1143 popl %eax
e7d3737e 1144 ret
fb52607a 1145END(ftrace_graph_caller)
caf4b323
FW		 1146
1147.globl return_to_handler
1148return_to_handler:
1149 pushl $0
1150 pushl %eax
1151 pushl %ecx
1152 pushl %edx
1153 call ftrace_return_to_handler
1154 movl %eax, 0xc(%esp)
1155 popl %edx
1156 popl %ecx
1157 popl %eax
1158 ret
e7d3737e 1159#endif
16444a8a 1160
bb152f53 1161.section .rodata,"a"
541054d9 1162#include "syscall_table_32.S"
1da177e4
LT		 1163
1164syscall_table_size=(.-sys_call_table)
d211af05
AH		 1165
1166/*
1167 * Some functions should be protected against kprobes
1168 */
1169 .pushsection .kprobes.text, "ax"
1170
1171ENTRY(page_fault)
1172 RING0_EC_FRAME
1173 pushl $do_page_fault
1174 CFI_ADJUST_CFA_OFFSET 4
1175 ALIGN
1176error_code:
ccbeed3a
TH		 1177 /* the function address is in %gs's slot on the stack */
1178 pushl %fs
1179 CFI_ADJUST_CFA_OFFSET 4
1180 /*CFI_REL_OFFSET fs, 0*/
d211af05
AH		 1181 pushl %es
1182 CFI_ADJUST_CFA_OFFSET 4
1183 /*CFI_REL_OFFSET es, 0*/
1184 pushl %ds
1185 CFI_ADJUST_CFA_OFFSET 4
1186 /*CFI_REL_OFFSET ds, 0*/
1187 pushl %eax
1188 CFI_ADJUST_CFA_OFFSET 4
1189 CFI_REL_OFFSET eax, 0
1190 pushl %ebp
1191 CFI_ADJUST_CFA_OFFSET 4
1192 CFI_REL_OFFSET ebp, 0
1193 pushl %edi
1194 CFI_ADJUST_CFA_OFFSET 4
1195 CFI_REL_OFFSET edi, 0
1196 pushl %esi
1197 CFI_ADJUST_CFA_OFFSET 4
1198 CFI_REL_OFFSET esi, 0
1199 pushl %edx
1200 CFI_ADJUST_CFA_OFFSET 4
1201 CFI_REL_OFFSET edx, 0
1202 pushl %ecx
1203 CFI_ADJUST_CFA_OFFSET 4
1204 CFI_REL_OFFSET ecx, 0
1205 pushl %ebx
1206 CFI_ADJUST_CFA_OFFSET 4
1207 CFI_REL_OFFSET ebx, 0
1208 cld
d211af05
AH		 1209 movl $(__KERNEL_PERCPU), %ecx
1210 movl %ecx, %fs
1211 UNWIND_ESPFIX_STACK
ccbeed3a
TH		 1212 GS_TO_REG %ecx
1213 movl PT_GS(%esp), %edi # get the function address
d211af05
AH		 1214 movl PT_ORIG_EAX(%esp), %edx # get the error code
1215 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
ccbeed3a
TH		 1216 REG_TO_PTGS %ecx
1217 SET_KERNEL_GS %ecx
d211af05
AH		 1218 movl $(__USER_DS), %ecx
1219 movl %ecx, %ds
1220 movl %ecx, %es
1221 TRACE_IRQS_OFF
1222 movl %esp,%eax # pt_regs pointer
1223 call *%edi
1224 jmp ret_from_exception
1225 CFI_ENDPROC
1226END(page_fault)
1227
1228/*
1229 * Debug traps and NMI can happen at the one SYSENTER instruction
1230 * that sets up the real kernel stack. Check here, since we can't
1231 * allow the wrong stack to be used.
1232 *
1233 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
1234 * already pushed 3 words if it hits on the sysenter instruction:
1235 * eflags, cs and eip.
1236 *
1237 * We just load the right stack, and push the three (known) values
1238 * by hand onto the new stack - while updating the return eip past
1239 * the instruction that would have done it for sysenter.
1240 */
f0d96110
TH		 1241.macro FIX_STACK offset ok label
1242 cmpw $__KERNEL_CS, 4(%esp)
1243 jne \ok
1244\label:
1245 movl TSS_sysenter_sp0 + \offset(%esp), %esp
1246 CFI_DEF_CFA esp, 0
1247 CFI_UNDEFINED eip
1248 pushfl
1249 CFI_ADJUST_CFA_OFFSET 4
1250 pushl $__KERNEL_CS
1251 CFI_ADJUST_CFA_OFFSET 4
1252 pushl $sysenter_past_esp
1253 CFI_ADJUST_CFA_OFFSET 4
d211af05 1254 CFI_REL_OFFSET eip, 0
f0d96110 1255.endm
d211af05
AH		 1256
1257ENTRY(debug)
1258 RING0_INT_FRAME
1259 cmpl $ia32_sysenter_target,(%esp)
1260 jne debug_stack_correct
f0d96110 1261 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
d211af05
AH		 1262debug_stack_correct:
1263 pushl $-1 # mark this as an int
1264 CFI_ADJUST_CFA_OFFSET 4
1265 SAVE_ALL
1266 TRACE_IRQS_OFF
1267 xorl %edx,%edx # error code 0
1268 movl %esp,%eax # pt_regs pointer
1269 call do_debug
1270 jmp ret_from_exception
1271 CFI_ENDPROC
1272END(debug)
1273
1274/*
1275 * NMI is doubly nasty. It can happen _while_ we're handling
1276 * a debug fault, and the debug fault hasn't yet been able to
1277 * clear up the stack. So we first check whether we got an
1278 * NMI on the sysenter entry path, but after that we need to
1279 * check whether we got an NMI on the debug path where the debug
1280 * fault happened on the sysenter path.
1281 */
1282ENTRY(nmi)
1283 RING0_INT_FRAME
1284 pushl %eax
1285 CFI_ADJUST_CFA_OFFSET 4
1286 movl %ss, %eax
1287 cmpw $__ESPFIX_SS, %ax
1288 popl %eax
1289 CFI_ADJUST_CFA_OFFSET -4
1290 je nmi_espfix_stack
1291 cmpl $ia32_sysenter_target,(%esp)
1292 je nmi_stack_fixup
1293 pushl %eax
1294 CFI_ADJUST_CFA_OFFSET 4
1295 movl %esp,%eax
1296 /* Do not access memory above the end of our stack page,
1297 * it might not exist.
1298 */
1299 andl $(THREAD_SIZE-1),%eax
1300 cmpl $(THREAD_SIZE-20),%eax
1301 popl %eax
1302 CFI_ADJUST_CFA_OFFSET -4
1303 jae nmi_stack_correct
1304 cmpl $ia32_sysenter_target,12(%esp)
1305 je nmi_debug_stack_check
1306nmi_stack_correct:
1307 /* We have a RING0_INT_FRAME here */
1308 pushl %eax
1309 CFI_ADJUST_CFA_OFFSET 4
1310 SAVE_ALL
d211af05
AH		 1311 xorl %edx,%edx # zero error code
1312 movl %esp,%eax # pt_regs pointer
1313 call do_nmi
1314 jmp restore_nocheck_notrace
1315 CFI_ENDPROC
1316
1317nmi_stack_fixup:
1318 RING0_INT_FRAME
f0d96110 1319 FIX_STACK 12, nmi_stack_correct, 1
d211af05
AH		 1320 jmp nmi_stack_correct
1321
1322nmi_debug_stack_check:
1323 /* We have a RING0_INT_FRAME here */
1324 cmpw $__KERNEL_CS,16(%esp)
1325 jne nmi_stack_correct
1326 cmpl $debug,(%esp)
1327 jb nmi_stack_correct
1328 cmpl $debug_esp_fix_insn,(%esp)
1329 ja nmi_stack_correct
f0d96110 1330 FIX_STACK 24, nmi_stack_correct, 1
d211af05
AH		 1331 jmp nmi_stack_correct
1332
1333nmi_espfix_stack:
1334 /* We have a RING0_INT_FRAME here.
1335 *
1336 * create the pointer to lss back
1337 */
1338 pushl %ss
1339 CFI_ADJUST_CFA_OFFSET 4
1340 pushl %esp
1341 CFI_ADJUST_CFA_OFFSET 4
1342 addw $4, (%esp)
1343 /* copy the iret frame of 12 bytes */
1344 .rept 3
1345 pushl 16(%esp)
1346 CFI_ADJUST_CFA_OFFSET 4
1347 .endr
1348 pushl %eax
1349 CFI_ADJUST_CFA_OFFSET 4
1350 SAVE_ALL
d211af05
AH		 1351 FIXUP_ESPFIX_STACK # %eax == %esp
1352 xorl %edx,%edx # zero error code
1353 call do_nmi
1354 RESTORE_REGS
1355 lss 12+4(%esp), %esp # back to espfix stack
1356 CFI_ADJUST_CFA_OFFSET -24
1357 jmp irq_return
1358 CFI_ENDPROC
1359END(nmi)
1360
1361ENTRY(int3)
1362 RING0_INT_FRAME
1363 pushl $-1 # mark this as an int
1364 CFI_ADJUST_CFA_OFFSET 4
1365 SAVE_ALL
1366 TRACE_IRQS_OFF
1367 xorl %edx,%edx # zero error code
1368 movl %esp,%eax # pt_regs pointer
1369 call do_int3
1370 jmp ret_from_exception
1371 CFI_ENDPROC
1372END(int3)
1373
1374ENTRY(general_protection)
1375 RING0_EC_FRAME
1376 pushl $do_general_protection
1377 CFI_ADJUST_CFA_OFFSET 4
1378 jmp error_code
1379 CFI_ENDPROC
1380END(general_protection)
1381
1382/*
1383 * End of kprobes section
1384 */
1385 .popsection
Linux 6.x block layer and io_uring tree(s)