projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
x86-32: Add new pt_regs stubs
[linux-2.6-block.git] / arch / x86 / kernel / entry_32.S
CommitLineData
1da177e4 1/*
1da177e4
LT		 2 *
3 * Copyright (C) 1991, 1992 Linus Torvalds
4 */
5
6/*
7 * entry.S contains the system-call and fault low-level handling routines.
8 * This also contains the timer-interrupt handler, as well as all interrupts
9 * and faults that can result in a task-switch.
10 *
11 * NOTE: This code handles signal-recognition, which happens every time
12 * after a timer-interrupt and after each system call.
13 *
14 * I changed all the .align's to 4 (16 byte alignment), as that's faster
15 * on a 486.
16 *
889f21ce 17 * Stack layout in 'syscall_exit':
1da177e4
LT		 18 * ptrace needs to have all regs on the stack.
19 * if the order here is changed, it needs to be
20 * updated in fork.c:copy_process, signal.c:do_signal,
21 * ptrace.c and ptrace.h
22 *
23 * 0(%esp) - %ebx
24 * 4(%esp) - %ecx
25 * 8(%esp) - %edx
26 * C(%esp) - %esi
27 * 10(%esp) - %edi
28 * 14(%esp) - %ebp
29 * 18(%esp) - %eax
30 * 1C(%esp) - %ds
31 * 20(%esp) - %es
464d1a78 32 * 24(%esp) - %fs
ccbeed3a
TH		 33 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS
34 * 2C(%esp) - orig_eax
35 * 30(%esp) - %eip
36 * 34(%esp) - %cs
37 * 38(%esp) - %eflags
38 * 3C(%esp) - %oldesp
39 * 40(%esp) - %oldss
1da177e4
LT		 40 *
41 * "current" is in register %ebx during any slow entries.
42 */
43
1da177e4
LT		 44#include <linux/linkage.h>
45#include <asm/thread_info.h>
55f327fa 46#include <asm/irqflags.h>
1da177e4
LT		 47#include <asm/errno.h>
48#include <asm/segment.h>
49#include <asm/smp.h>
0341c14d 50#include <asm/page_types.h>
be44d2aa 51#include <asm/percpu.h>
fe7cacc1 52#include <asm/dwarf2.h>
ab68ed98 53#include <asm/processor-flags.h>
395a59d0 54#include <asm/ftrace.h>
9b7dc567 55#include <asm/irq_vectors.h>
1da177e4 56
af0575bb
RM		 57/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
58#include <linux/elf-em.h>
59#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE)
60#define __AUDIT_ARCH_LE 0x40000000
61
62#ifndef CONFIG_AUDITSYSCALL
63#define sysenter_audit syscall_trace_entry
64#define sysexit_audit syscall_exit_work
65#endif
66
139ec7c4
RR		 67/*
68 * We use macros for low-level operations which need to be overridden
69 * for paravirtualization. The following will never clobber any registers:
70 * INTERRUPT_RETURN (aka. "iret")
71 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
d75cd22f 72 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
139ec7c4
RR		 73 *
74 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
75 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
76 * Allowing a register to be clobbered can shrink the paravirt replacement
77 * enough to patch inline, increasing performance.
78 */
79
1da177e4
LT		 80#define nr_syscalls ((syscall_table_size)/4)
81
1da177e4 82#ifdef CONFIG_PREEMPT
139ec7c4 83#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
1da177e4 84#else
139ec7c4 85#define preempt_stop(clobbers)
2e04bc76 86#define resume_kernel restore_all
1da177e4
LT		 87#endif
88
55f327fa
IM		 89.macro TRACE_IRQS_IRET
90#ifdef CONFIG_TRACE_IRQFLAGS
ab68ed98 91 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off?
55f327fa
IM		 92 jz 1f
93 TRACE_IRQS_ON
941:
95#endif
96.endm
97
4031ff38
AG		 98#ifdef CONFIG_VM86
99#define resume_userspace_sig check_userspace
100#else
101#define resume_userspace_sig resume_userspace
102#endif
103
ccbeed3a
TH		 104/*
105 * User gs save/restore
106 *
107 * %gs is used for userland TLS and kernel only uses it for stack
108 * canary which is required to be at %gs:20 by gcc. Read the comment
109 * at the top of stackprotector.h for more info.
110 *
111 * Local labels 98 and 99 are used.
112 */
113#ifdef CONFIG_X86_32_LAZY_GS
114
115 /* unfortunately push/pop can't be no-op */
116.macro PUSH_GS
117 pushl $0
118 CFI_ADJUST_CFA_OFFSET 4
119.endm
120.macro POP_GS pop=0
121 addl $(4 + \pop), %esp
122 CFI_ADJUST_CFA_OFFSET -(4 + \pop)
123.endm
124.macro POP_GS_EX
125.endm
126
127 /* all the rest are no-op */
128.macro PTGS_TO_GS
129.endm
130.macro PTGS_TO_GS_EX
131.endm
132.macro GS_TO_REG reg
133.endm
134.macro REG_TO_PTGS reg
135.endm
136.macro SET_KERNEL_GS reg
137.endm
138
139#else /* CONFIG_X86_32_LAZY_GS */
140
141.macro PUSH_GS
142 pushl %gs
143 CFI_ADJUST_CFA_OFFSET 4
144 /*CFI_REL_OFFSET gs, 0*/
145.endm
146
147.macro POP_GS pop=0
14898: popl %gs
149 CFI_ADJUST_CFA_OFFSET -4
150 /*CFI_RESTORE gs*/
151 .if \pop <> 0
152 add $\pop, %esp
153 CFI_ADJUST_CFA_OFFSET -\pop
154 .endif
155.endm
156.macro POP_GS_EX
157.pushsection .fixup, "ax"
15899: movl $0, (%esp)
159 jmp 98b
160.section __ex_table, "a"
161 .align 4
162 .long 98b, 99b
163.popsection
164.endm
165
166.macro PTGS_TO_GS
16798: mov PT_GS(%esp), %gs
168.endm
169.macro PTGS_TO_GS_EX
170.pushsection .fixup, "ax"
17199: movl $0, PT_GS(%esp)
172 jmp 98b
173.section __ex_table, "a"
174 .align 4
175 .long 98b, 99b
176.popsection
177.endm
178
179.macro GS_TO_REG reg
180 movl %gs, \reg
181 /*CFI_REGISTER gs, \reg*/
182.endm
183.macro REG_TO_PTGS reg
184 movl \reg, PT_GS(%esp)
185 /*CFI_REL_OFFSET gs, PT_GS*/
186.endm
187.macro SET_KERNEL_GS reg
60a5317f 188 movl $(__KERNEL_STACK_CANARY), \reg
ccbeed3a
TH		 189 movl \reg, %gs
190.endm
191
192#endif /* CONFIG_X86_32_LAZY_GS */
193
f0d96110
TH		 194.macro SAVE_ALL
195 cld
ccbeed3a 196 PUSH_GS
f0d96110
TH		 197 pushl %fs
198 CFI_ADJUST_CFA_OFFSET 4
199 /*CFI_REL_OFFSET fs, 0;*/
200 pushl %es
201 CFI_ADJUST_CFA_OFFSET 4
202 /*CFI_REL_OFFSET es, 0;*/
203 pushl %ds
204 CFI_ADJUST_CFA_OFFSET 4
205 /*CFI_REL_OFFSET ds, 0;*/
206 pushl %eax
207 CFI_ADJUST_CFA_OFFSET 4
208 CFI_REL_OFFSET eax, 0
209 pushl %ebp
210 CFI_ADJUST_CFA_OFFSET 4
211 CFI_REL_OFFSET ebp, 0
212 pushl %edi
213 CFI_ADJUST_CFA_OFFSET 4
214 CFI_REL_OFFSET edi, 0
215 pushl %esi
216 CFI_ADJUST_CFA_OFFSET 4
217 CFI_REL_OFFSET esi, 0
218 pushl %edx
219 CFI_ADJUST_CFA_OFFSET 4
220 CFI_REL_OFFSET edx, 0
221 pushl %ecx
222 CFI_ADJUST_CFA_OFFSET 4
223 CFI_REL_OFFSET ecx, 0
224 pushl %ebx
225 CFI_ADJUST_CFA_OFFSET 4
226 CFI_REL_OFFSET ebx, 0
227 movl $(__USER_DS), %edx
228 movl %edx, %ds
229 movl %edx, %es
230 movl $(__KERNEL_PERCPU), %edx
464d1a78 231 movl %edx, %fs
ccbeed3a 232 SET_KERNEL_GS %edx
f0d96110 233.endm
1da177e4 234
f0d96110
TH		 235.macro RESTORE_INT_REGS
236 popl %ebx
237 CFI_ADJUST_CFA_OFFSET -4
238 CFI_RESTORE ebx
239 popl %ecx
240 CFI_ADJUST_CFA_OFFSET -4
241 CFI_RESTORE ecx
242 popl %edx
243 CFI_ADJUST_CFA_OFFSET -4
244 CFI_RESTORE edx
245 popl %esi
246 CFI_ADJUST_CFA_OFFSET -4
247 CFI_RESTORE esi
248 popl %edi
249 CFI_ADJUST_CFA_OFFSET -4
250 CFI_RESTORE edi
251 popl %ebp
252 CFI_ADJUST_CFA_OFFSET -4
253 CFI_RESTORE ebp
254 popl %eax
255 CFI_ADJUST_CFA_OFFSET -4
fe7cacc1 256 CFI_RESTORE eax
f0d96110 257.endm
1da177e4 258
ccbeed3a 259.macro RESTORE_REGS pop=0
f0d96110
TH		 260 RESTORE_INT_REGS
2611: popl %ds
262 CFI_ADJUST_CFA_OFFSET -4
263 /*CFI_RESTORE ds;*/
2642: popl %es
265 CFI_ADJUST_CFA_OFFSET -4
266 /*CFI_RESTORE es;*/
2673: popl %fs
268 CFI_ADJUST_CFA_OFFSET -4
269 /*CFI_RESTORE fs;*/
ccbeed3a 270 POP_GS \pop
f0d96110
TH		 271.pushsection .fixup, "ax"
2724: movl $0, (%esp)
273 jmp 1b
2745: movl $0, (%esp)
275 jmp 2b
2766: movl $0, (%esp)
277 jmp 3b
278.section __ex_table, "a"
279 .align 4
280 .long 1b, 4b
281 .long 2b, 5b
282 .long 3b, 6b
f95d47ca 283.popsection
ccbeed3a 284 POP_GS_EX
f0d96110 285.endm
1da177e4 286
f0d96110
TH		 287.macro RING0_INT_FRAME
288 CFI_STARTPROC simple
289 CFI_SIGNAL_FRAME
290 CFI_DEF_CFA esp, 3*4
291 /*CFI_OFFSET cs, -2*4;*/
fe7cacc1 292 CFI_OFFSET eip, -3*4
f0d96110 293.endm
fe7cacc1 294
f0d96110
TH		 295.macro RING0_EC_FRAME
296 CFI_STARTPROC simple
297 CFI_SIGNAL_FRAME
298 CFI_DEF_CFA esp, 4*4
299 /*CFI_OFFSET cs, -2*4;*/
fe7cacc1 300 CFI_OFFSET eip, -3*4
f0d96110 301.endm
fe7cacc1 302
f0d96110
TH		 303.macro RING0_PTREGS_FRAME
304 CFI_STARTPROC simple
305 CFI_SIGNAL_FRAME
306 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX
307 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/
308 CFI_OFFSET eip, PT_EIP-PT_OLDESP
309 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/
310 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/
311 CFI_OFFSET eax, PT_EAX-PT_OLDESP
312 CFI_OFFSET ebp, PT_EBP-PT_OLDESP
313 CFI_OFFSET edi, PT_EDI-PT_OLDESP
314 CFI_OFFSET esi, PT_ESI-PT_OLDESP
315 CFI_OFFSET edx, PT_EDX-PT_OLDESP
316 CFI_OFFSET ecx, PT_ECX-PT_OLDESP
eb5b7b9d 317 CFI_OFFSET ebx, PT_EBX-PT_OLDESP
f0d96110 318.endm
1da177e4
LT		 319
320ENTRY(ret_from_fork)
fe7cacc1 321 CFI_STARTPROC
1da177e4 322 pushl %eax
25d7dfda 323 CFI_ADJUST_CFA_OFFSET 4
1da177e4
LT		 324 call schedule_tail
325 GET_THREAD_INFO(%ebp)
326 popl %eax
fe7cacc1 327 CFI_ADJUST_CFA_OFFSET -4
47a5c6fa
LT		 328 pushl $0x0202 # Reset kernel eflags
329 CFI_ADJUST_CFA_OFFSET 4
330 popfl
331 CFI_ADJUST_CFA_OFFSET -4
1da177e4 332 jmp syscall_exit
fe7cacc1 333 CFI_ENDPROC
47a55cd7 334END(ret_from_fork)
1da177e4 335
a00e817f
MH		 336/*
337 * Interrupt exit functions should be protected against kprobes
338 */
339 .pushsection .kprobes.text, "ax"
1da177e4
LT		 340/*
341 * Return to user mode is not as complex as all this looks,
342 * but we want the default path for a system call return to
343 * go as quickly as possible which is why some of this is
344 * less clear than it otherwise should be.
345 */
346
347 # userspace resumption stub bypassing syscall exit tracing
348 ALIGN
fe7cacc1 349 RING0_PTREGS_FRAME
1da177e4 350ret_from_exception:
139ec7c4 351 preempt_stop(CLBR_ANY)
1da177e4
LT		 352ret_from_intr:
353 GET_THREAD_INFO(%ebp)
4031ff38 354check_userspace:
eb5b7b9d
JF		 355 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
356 movb PT_CS(%esp), %al
ab68ed98 357 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
78be3706
RR		 358 cmpl $USER_RPL, %eax
359 jb resume_kernel # not returning to v8086 or userspace
f95d47ca 360
1da177e4 361ENTRY(resume_userspace)
c7e872e7 362 LOCKDEP_SYS_EXIT
139ec7c4 363 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
1da177e4
LT		 364 # setting need_resched or sigpending
365 # between sampling and the iret
e32e58a9 366 TRACE_IRQS_OFF
1da177e4
LT		 367 movl TI_flags(%ebp), %ecx
368 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
369 # int/exception return?
370 jne work_pending
371 jmp restore_all
47a55cd7 372END(ret_from_exception)
1da177e4
LT		 373
374#ifdef CONFIG_PREEMPT
375ENTRY(resume_kernel)
139ec7c4 376 DISABLE_INTERRUPTS(CLBR_ANY)
1da177e4 377 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
2e04bc76 378 jnz restore_all
1da177e4
LT		 379need_resched:
380 movl TI_flags(%ebp), %ecx # need_resched set ?
381 testb $_TIF_NEED_RESCHED, %cl
382 jz restore_all
ab68ed98 383 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ?
1da177e4
LT		 384 jz restore_all
385 call preempt_schedule_irq
386 jmp need_resched
47a55cd7 387END(resume_kernel)
1da177e4 388#endif
fe7cacc1 389 CFI_ENDPROC
a00e817f
MH		 390/*
391 * End of kprobes section
392 */
393 .popsection
1da177e4
LT		 394
395/* SYSENTER_RETURN points to after the "sysenter" instruction in
396 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
397
398 # sysenter call handler stub
0aa97fb2 399ENTRY(ia32_sysenter_target)
fe7cacc1 400 CFI_STARTPROC simple
adf14236 401 CFI_SIGNAL_FRAME
fe7cacc1
JB		 402 CFI_DEF_CFA esp, 0
403 CFI_REGISTER esp, ebp
faca6227 404 movl TSS_sysenter_sp0(%esp),%esp
1da177e4 405sysenter_past_esp:
55f327fa 406 /*
d93c870b
JF		 407 * Interrupts are disabled here, but we can't trace it until
408 * enough kernel state to call TRACE_IRQS_OFF can be called - but
409 * we immediately enable interrupts at that point anyway.
55f327fa 410 */
1da177e4 411 pushl $(__USER_DS)
fe7cacc1
JB		 412 CFI_ADJUST_CFA_OFFSET 4
413 /*CFI_REL_OFFSET ss, 0*/
1da177e4 414 pushl %ebp
fe7cacc1
JB		 415 CFI_ADJUST_CFA_OFFSET 4
416 CFI_REL_OFFSET esp, 0
1da177e4 417 pushfl
d93c870b 418 orl $X86_EFLAGS_IF, (%esp)
fe7cacc1 419 CFI_ADJUST_CFA_OFFSET 4
1da177e4 420 pushl $(__USER_CS)
fe7cacc1
JB		 421 CFI_ADJUST_CFA_OFFSET 4
422 /*CFI_REL_OFFSET cs, 0*/
e6e5494c
IM		 423 /*
424 * Push current_thread_info()->sysenter_return to the stack.
425 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
426 * pushed above; +8 corresponds to copy_thread's esp0 setting.
427 */
428 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
fe7cacc1
JB		 429 CFI_ADJUST_CFA_OFFSET 4
430 CFI_REL_OFFSET eip, 0
1da177e4 431
d93c870b
JF		 432 pushl %eax
433 CFI_ADJUST_CFA_OFFSET 4
434 SAVE_ALL
435 ENABLE_INTERRUPTS(CLBR_NONE)
436
1da177e4
LT		 437/*
438 * Load the potential sixth argument from user stack.
439 * Careful about security.
440 */
441 cmpl $__PAGE_OFFSET-3,%ebp
442 jae syscall_fault
4431: movl (%ebp),%ebp
d93c870b 444 movl %ebp,PT_EBP(%esp)
1da177e4
LT		 445.section __ex_table,"a"
446 .align 4
447 .long 1b,syscall_fault
448.previous
449
1da177e4
LT		 450 GET_THREAD_INFO(%ebp)
451
88200bc2 452 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
af0575bb
RM		 453 jnz sysenter_audit
454sysenter_do_call:
1da177e4
LT		 455 cmpl $(nr_syscalls), %eax
456 jae syscall_badsys
457 call *sys_call_table(,%eax,4)
eb5b7b9d 458 movl %eax,PT_EAX(%esp)
c7e872e7 459 LOCKDEP_SYS_EXIT
42c24fa2 460 DISABLE_INTERRUPTS(CLBR_ANY)
55f327fa 461 TRACE_IRQS_OFF
1da177e4 462 movl TI_flags(%ebp), %ecx
88200bc2 463 testl $_TIF_ALLWORK_MASK, %ecx
af0575bb
RM		 464 jne sysexit_audit
465sysenter_exit:
1da177e4 466/* if something modifies registers it must also disable sysexit */
eb5b7b9d
JF		 467 movl PT_EIP(%esp), %edx
468 movl PT_OLDESP(%esp), %ecx
1da177e4 469 xorl %ebp,%ebp
55f327fa 470 TRACE_IRQS_ON
464d1a78 4711: mov PT_FS(%esp), %fs
ccbeed3a 472 PTGS_TO_GS
d75cd22f 473 ENABLE_INTERRUPTS_SYSEXIT
af0575bb
RM		 474
475#ifdef CONFIG_AUDITSYSCALL
476sysenter_audit:
88200bc2 477 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
af0575bb
RM		 478 jnz syscall_trace_entry
479 addl $4,%esp
480 CFI_ADJUST_CFA_OFFSET -4
481 /* %esi already in 8(%esp) 6th arg: 4th syscall arg */
482 /* %edx already in 4(%esp) 5th arg: 3rd syscall arg */
483 /* %ecx already in 0(%esp) 4th arg: 2nd syscall arg */
484 movl %ebx,%ecx /* 3rd arg: 1st syscall arg */
485 movl %eax,%edx /* 2nd arg: syscall number */
486 movl $AUDIT_ARCH_I386,%eax /* 1st arg: audit arch */
487 call audit_syscall_entry
488 pushl %ebx
489 CFI_ADJUST_CFA_OFFSET 4
490 movl PT_EAX(%esp),%eax /* reload syscall number */
491 jmp sysenter_do_call
492
493sysexit_audit:
88200bc2 494 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
af0575bb
RM		 495 jne syscall_exit_work
496 TRACE_IRQS_ON
497 ENABLE_INTERRUPTS(CLBR_ANY)
498 movl %eax,%edx /* second arg, syscall return value */
499 cmpl $0,%eax /* is it < 0? */
500 setl %al /* 1 if so, 0 if not */
501 movzbl %al,%eax /* zero-extend that */
502 inc %eax /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
503 call audit_syscall_exit
504 DISABLE_INTERRUPTS(CLBR_ANY)
505 TRACE_IRQS_OFF
506 movl TI_flags(%ebp), %ecx
88200bc2 507 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
af0575bb
RM		 508 jne syscall_exit_work
509 movl PT_EAX(%esp),%eax /* reload syscall return value */
510 jmp sysenter_exit
511#endif
512
fe7cacc1 513 CFI_ENDPROC
f95d47ca 514.pushsection .fixup,"ax"
464d1a78 5152: movl $0,PT_FS(%esp)
f95d47ca
JF		 516 jmp 1b
517.section __ex_table,"a"
518 .align 4
519 .long 1b,2b
520.popsection
ccbeed3a 521 PTGS_TO_GS_EX
0aa97fb2 522ENDPROC(ia32_sysenter_target)
1da177e4 523
a00e817f
MH		 524/*
525 * syscall stub including irq exit should be protected against kprobes
526 */
527 .pushsection .kprobes.text, "ax"
1da177e4
LT		 528 # system call handler stub
529ENTRY(system_call)
fe7cacc1 530 RING0_INT_FRAME # can't unwind into user space anyway
1da177e4 531 pushl %eax # save orig_eax
fe7cacc1 532 CFI_ADJUST_CFA_OFFSET 4
1da177e4
LT		 533 SAVE_ALL
534 GET_THREAD_INFO(%ebp)
ed75e8d5 535 # system call tracing in operation / emulation
88200bc2 536 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
1da177e4
LT		 537 jnz syscall_trace_entry
538 cmpl $(nr_syscalls), %eax
539 jae syscall_badsys
540syscall_call:
541 call *sys_call_table(,%eax,4)
eb5b7b9d 542 movl %eax,PT_EAX(%esp) # store the return value
1da177e4 543syscall_exit:
c7e872e7 544 LOCKDEP_SYS_EXIT
139ec7c4 545 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
1da177e4
LT		 546 # setting need_resched or sigpending
547 # between sampling and the iret
55f327fa 548 TRACE_IRQS_OFF
1da177e4 549 movl TI_flags(%ebp), %ecx
88200bc2 550 testl $_TIF_ALLWORK_MASK, %ecx # current->work
1da177e4
LT		 551 jne syscall_exit_work
552
553restore_all:
2e04bc76
AH		 554 TRACE_IRQS_IRET
555restore_all_notrace:
eb5b7b9d
JF		 556 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
557 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
5df24082
SS		 558 # are returning to the kernel.
559 # See comments in process.c:copy_thread() for details.
eb5b7b9d
JF		 560 movb PT_OLDSS(%esp), %ah
561 movb PT_CS(%esp), %al
ab68ed98 562 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
78be3706 563 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
fe7cacc1 564 CFI_REMEMBER_STATE
1da177e4
LT		 565 je ldt_ss # returning to user-space with LDT SS
566restore_nocheck:
ccbeed3a 567 RESTORE_REGS 4 # skip orig_eax/error_code
fe7cacc1 568 CFI_ADJUST_CFA_OFFSET -4
f7f3d791 569irq_return:
3701d863 570 INTERRUPT_RETURN
1da177e4 571.section .fixup,"ax"
90e9f536 572ENTRY(iret_exc)
a879cbbb
LT		 573 pushl $0 # no error code
574 pushl $do_iret_error
575 jmp error_code
1da177e4
LT		 576.previous
577.section __ex_table,"a"
578 .align 4
3701d863 579 .long irq_return,iret_exc
1da177e4
LT		 580.previous
581
fe7cacc1 582 CFI_RESTORE_STATE
1da177e4 583ldt_ss:
eb5b7b9d 584 larl PT_OLDSS(%esp), %eax
1da177e4
LT		 585 jnz restore_nocheck
586 testl $0x00400000, %eax # returning to 32bit stack?
587 jnz restore_nocheck # allright, normal return
d3561b7f
RR		 588
589#ifdef CONFIG_PARAVIRT
590 /*
591 * The kernel can't run on a non-flat stack if paravirt mode
592 * is active. Rather than try to fixup the high bits of
593 * ESP, bypass this code entirely. This may break DOSemu
594 * and/or Wine support in a paravirt VM, although the option
595 * is still available to implement the setting of the high
596 * 16-bits in the INTERRUPT_RETURN paravirt-op.
597 */
93b1eab3 598 cmpl $0, pv_info+PARAVIRT_enabled
d3561b7f
RR		 599 jne restore_nocheck
600#endif
601
dc4c2a0a
AH		 602/*
603 * Setup and switch to ESPFIX stack
604 *
605 * We're returning to userspace with a 16 bit stack. The CPU will not
606 * restore the high word of ESP for us on executing iret... This is an
607 * "official" bug of all the x86-compatible CPUs, which we can work
608 * around to make dosemu and wine happy. We do this by preloading the
609 * high word of ESP with the high word of the userspace ESP while
610 * compensating for the offset by changing to the ESPFIX segment with
611 * a base address that matches for the difference.
612 */
613 mov %esp, %edx /* load kernel esp */
614 mov PT_OLDESP(%esp), %eax /* load userspace esp */
615 mov %dx, %ax /* eax: new kernel esp */
616 sub %eax, %edx /* offset (low word is 0) */
617 PER_CPU(gdt_page, %ebx)
618 shr $16, %edx
619 mov %dl, GDT_ENTRY_ESPFIX_SS * 8 + 4(%ebx) /* bits 16..23 */
620 mov %dh, GDT_ENTRY_ESPFIX_SS * 8 + 7(%ebx) /* bits 24..31 */
be44d2aa
SS		 621 pushl $__ESPFIX_SS
622 CFI_ADJUST_CFA_OFFSET 4
dc4c2a0a 623 push %eax /* new kernel esp */
be44d2aa 624 CFI_ADJUST_CFA_OFFSET 4
2e04bc76
AH		 625 /* Disable interrupts, but do not irqtrace this section: we
626 * will soon execute iret and the tracer was already set to
627 * the irqstate after the iret */
139ec7c4 628 DISABLE_INTERRUPTS(CLBR_EAX)
dc4c2a0a 629 lss (%esp), %esp /* switch to espfix segment */
be44d2aa
SS		 630 CFI_ADJUST_CFA_OFFSET -8
631 jmp restore_nocheck
fe7cacc1 632 CFI_ENDPROC
47a55cd7 633ENDPROC(system_call)
1da177e4
LT		 634
635 # perform work that needs to be done immediately before resumption
636 ALIGN
fe7cacc1 637 RING0_PTREGS_FRAME # can't unwind into user space anyway
1da177e4
LT		 638work_pending:
639 testb $_TIF_NEED_RESCHED, %cl
640 jz work_notifysig
641work_resched:
642 call schedule
c7e872e7 643 LOCKDEP_SYS_EXIT
139ec7c4 644 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
1da177e4
LT		 645 # setting need_resched or sigpending
646 # between sampling and the iret
55f327fa 647 TRACE_IRQS_OFF
1da177e4
LT		 648 movl TI_flags(%ebp), %ecx
649 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
650 # than syscall tracing?
651 jz restore_all
652 testb $_TIF_NEED_RESCHED, %cl
653 jnz work_resched
654
655work_notifysig: # deal with pending signals and
656 # notify-resume requests
74b47a78 657#ifdef CONFIG_VM86
ab68ed98 658 testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
1da177e4
LT		 659 movl %esp, %eax
660 jne work_notifysig_v86 # returning to kernel-space or
661 # vm86-space
662 xorl %edx, %edx
663 call do_notify_resume
4031ff38 664 jmp resume_userspace_sig
1da177e4
LT		 665
666 ALIGN
667work_notifysig_v86:
668 pushl %ecx # save ti_flags for do_notify_resume
fe7cacc1 669 CFI_ADJUST_CFA_OFFSET 4
1da177e4
LT		 670 call save_v86_state # %eax contains pt_regs pointer
671 popl %ecx
fe7cacc1 672 CFI_ADJUST_CFA_OFFSET -4
1da177e4 673 movl %eax, %esp
74b47a78
JK		 674#else
675 movl %esp, %eax
676#endif
1da177e4
LT		 677 xorl %edx, %edx
678 call do_notify_resume
4031ff38 679 jmp resume_userspace_sig
47a55cd7 680END(work_pending)
1da177e4
LT		 681
682 # perform syscall exit tracing
683 ALIGN
684syscall_trace_entry:
eb5b7b9d 685 movl $-ENOSYS,PT_EAX(%esp)
1da177e4 686 movl %esp, %eax
d4d67150
RM		 687 call syscall_trace_enter
688 /* What it returned is what we'll actually use. */
1da177e4
LT		 689 cmpl $(nr_syscalls), %eax
690 jnae syscall_call
691 jmp syscall_exit
47a55cd7 692END(syscall_trace_entry)
1da177e4
LT		 693
694 # perform syscall exit tracing
695 ALIGN
696syscall_exit_work:
88200bc2 697 testl $_TIF_WORK_SYSCALL_EXIT, %ecx
1da177e4 698 jz work_pending
55f327fa 699 TRACE_IRQS_ON
d4d67150 700 ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call
1da177e4
LT		 701 # schedule() instead
702 movl %esp, %eax
d4d67150 703 call syscall_trace_leave
1da177e4 704 jmp resume_userspace
47a55cd7 705END(syscall_exit_work)
fe7cacc1 706 CFI_ENDPROC
1da177e4 707
fe7cacc1 708 RING0_INT_FRAME # can't unwind into user space anyway
1da177e4 709syscall_fault:
1da177e4 710 GET_THREAD_INFO(%ebp)
eb5b7b9d 711 movl $-EFAULT,PT_EAX(%esp)
1da177e4 712 jmp resume_userspace
47a55cd7 713END(syscall_fault)
1da177e4 714
1da177e4 715syscall_badsys:
eb5b7b9d 716 movl $-ENOSYS,PT_EAX(%esp)
1da177e4 717 jmp resume_userspace
47a55cd7 718END(syscall_badsys)
fe7cacc1 719 CFI_ENDPROC
a00e817f
MH		 720/*
721 * End of kprobes section
722 */
723 .popsection
1da177e4 724
253f29a4
BG		 725/*
726 * System calls that need a pt_regs pointer.
727 */
e258e4e0 728#define PTREGSCALL0(name) \
253f29a4
BG		 729 ALIGN; \
730ptregs_##name: \
731 leal 4(%esp),%eax; \
732 jmp sys_##name;
733
e258e4e0
BG		 734#define PTREGSCALL1(name) \
735 ALIGN; \
736ptregs_##name: \
737 leal 4(%esp),%edx; \
738 movl PT_EBX(%edx),%eax; \
739 jmp sys_##name;
740
741#define PTREGSCALL2(name) \
742 ALIGN; \
743ptregs_##name: \
744 leal 4(%esp),%ecx; \
745 movl PT_ECX(%ecx),%edx; \
746 movl PT_EBX(%ecx),%eax; \
747 jmp sys_##name;
748
749#define PTREGSCALL3(name) \
750 ALIGN; \
751ptregs_##name: \
752 leal 4(%esp),%eax; \
753 pushl %eax; \
754 movl PT_EDX(%eax),%ecx; \
755 movl PT_ECX(%eax),%edx; \
756 movl PT_EBX(%eax),%eax; \
757 call sys_##name; \
758 addl $4,%esp; \
759 ret
760
761PTREGSCALL0(iopl)
762PTREGSCALL0(fork)
763PTREGSCALL0(clone)
764PTREGSCALL0(vfork)
765PTREGSCALL0(execve)
766PTREGSCALL0(sigaltstack)
767PTREGSCALL0(sigreturn)
768PTREGSCALL0(rt_sigreturn)
769PTREGSCALL0(vm86)
770PTREGSCALL0(vm86old)
253f29a4 771
f0d96110 772.macro FIXUP_ESPFIX_STACK
dc4c2a0a
AH		 773/*
774 * Switch back for ESPFIX stack to the normal zerobased stack
775 *
776 * We can't call C functions using the ESPFIX stack. This code reads
777 * the high word of the segment base from the GDT and swiches to the
778 * normal stack and adjusts ESP with the matching offset.
779 */
780 /* fixup the stack */
f0d96110 781 PER_CPU(gdt_page, %ebx)
dc4c2a0a
AH		 782 mov GDT_ENTRY_ESPFIX_SS * 8 + 4(%ebx), %al /* bits 16..23 */
783 mov GDT_ENTRY_ESPFIX_SS * 8 + 7(%ebx), %ah /* bits 24..31 */
784 shl $16, %eax
785 addl %esp, %eax /* the adjusted stack pointer */
f0d96110
TH		 786 pushl $__KERNEL_DS
787 CFI_ADJUST_CFA_OFFSET 4
788 pushl %eax
789 CFI_ADJUST_CFA_OFFSET 4
dc4c2a0a 790 lss (%esp), %esp /* switch to the normal stack segment */
f0d96110
TH		 791 CFI_ADJUST_CFA_OFFSET -8
792.endm
793.macro UNWIND_ESPFIX_STACK
794 movl %ss, %eax
795 /* see if on espfix stack */
796 cmpw $__ESPFIX_SS, %ax
797 jne 27f
798 movl $__KERNEL_DS, %eax
799 movl %eax, %ds
800 movl %eax, %es
801 /* switch to normal stack */
802 FIXUP_ESPFIX_STACK
80327:
804.endm
1da177e4
LT		 805
806/*
b7c6244f
PA		 807 * Build the entry stubs and pointer table with some assembler magic.
808 * We pack 7 stubs into a single 32-byte chunk, which will fit in a
809 * single cache line on all modern x86 implementations.
1da177e4 810 */
4687518c 811.section .init.rodata,"a"
1da177e4
LT		 812ENTRY(interrupt)
813.text
b7c6244f
PA		 814 .p2align 5
815 .p2align CONFIG_X86_L1_CACHE_SHIFT
1da177e4 816ENTRY(irq_entries_start)
fe7cacc1 817 RING0_INT_FRAME
4687518c 818vector=FIRST_EXTERNAL_VECTOR
b7c6244f
PA		 819.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
820 .balign 32
821 .rept 7
822 .if vector < NR_VECTORS
8665596e 823 .if vector <> FIRST_EXTERNAL_VECTOR
fe7cacc1 824 CFI_ADJUST_CFA_OFFSET -4
b7c6244f
PA		 825 .endif
8261: pushl $(~vector+0x80) /* Note: always in signed byte range */
fe7cacc1 827 CFI_ADJUST_CFA_OFFSET 4
8665596e 828 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
b7c6244f
PA		 829 jmp 2f
830 .endif
831 .previous
1da177e4 832 .long 1b
b7c6244f 833 .text
1da177e4 834vector=vector+1
b7c6244f
PA		 835 .endif
836 .endr
8372: jmp common_interrupt
1da177e4 838.endr
47a55cd7
JB		 839END(irq_entries_start)
840
841.previous
842END(interrupt)
843.previous
1da177e4 844
55f327fa
IM		 845/*
846 * the CPU automatically disables interrupts when executing an IRQ vector,
847 * so IRQ-flags tracing has to follow that:
848 */
b7c6244f 849 .p2align CONFIG_X86_L1_CACHE_SHIFT
1da177e4 850common_interrupt:
b7c6244f 851 addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */
1da177e4 852 SAVE_ALL
55f327fa 853 TRACE_IRQS_OFF
1da177e4
LT		 854 movl %esp,%eax
855 call do_IRQ
856 jmp ret_from_intr
47a55cd7 857ENDPROC(common_interrupt)
fe7cacc1 858 CFI_ENDPROC
1da177e4 859
a00e817f
MH		 860/*
861 * Irq entries should be protected against kprobes
862 */
863 .pushsection .kprobes.text, "ax"
02cf94c3 864#define BUILD_INTERRUPT3(name, nr, fn) \
1da177e4 865ENTRY(name) \
fe7cacc1 866 RING0_INT_FRAME; \
19eadf98 867 pushl $~(nr); \
fe7cacc1
JB		 868 CFI_ADJUST_CFA_OFFSET 4; \
869 SAVE_ALL; \
55f327fa 870 TRACE_IRQS_OFF \
1da177e4 871 movl %esp,%eax; \
02cf94c3 872 call fn; \
55f327fa 873 jmp ret_from_intr; \
47a55cd7
JB		 874 CFI_ENDPROC; \
875ENDPROC(name)
1da177e4 876
02cf94c3
TH		 877#define BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(name, nr, smp_##name)
878
1da177e4 879/* The include is where all of the SMP etc. interrupts come from */
1164dd00 880#include <asm/entry_arch.h>
1da177e4 881
1da177e4 882ENTRY(coprocessor_error)
fe7cacc1 883 RING0_INT_FRAME
1da177e4 884 pushl $0
fe7cacc1 885 CFI_ADJUST_CFA_OFFSET 4
1da177e4 886 pushl $do_coprocessor_error
fe7cacc1 887 CFI_ADJUST_CFA_OFFSET 4
1da177e4 888 jmp error_code
fe7cacc1 889 CFI_ENDPROC
47a55cd7 890END(coprocessor_error)
1da177e4
LT		 891
892ENTRY(simd_coprocessor_error)
fe7cacc1 893 RING0_INT_FRAME
1da177e4 894 pushl $0
fe7cacc1 895 CFI_ADJUST_CFA_OFFSET 4
1da177e4 896 pushl $do_simd_coprocessor_error
fe7cacc1 897 CFI_ADJUST_CFA_OFFSET 4
1da177e4 898 jmp error_code
fe7cacc1 899 CFI_ENDPROC
47a55cd7 900END(simd_coprocessor_error)
1da177e4
LT		 901
902ENTRY(device_not_available)
fe7cacc1 903 RING0_INT_FRAME
1da177e4 904 pushl $-1 # mark this as an int
fe7cacc1 905 CFI_ADJUST_CFA_OFFSET 4
7643e9b9 906 pushl $do_device_not_available
fe7cacc1 907 CFI_ADJUST_CFA_OFFSET 4
7643e9b9 908 jmp error_code
fe7cacc1 909 CFI_ENDPROC
47a55cd7 910END(device_not_available)
1da177e4 911
d3561b7f
RR		 912#ifdef CONFIG_PARAVIRT
913ENTRY(native_iret)
3701d863 914 iret
d3561b7f
RR		 915.section __ex_table,"a"
916 .align 4
3701d863 917 .long native_iret, iret_exc
d3561b7f 918.previous
47a55cd7 919END(native_iret)
d3561b7f 920
d75cd22f 921ENTRY(native_irq_enable_sysexit)
d3561b7f
RR		 922 sti
923 sysexit
d75cd22f 924END(native_irq_enable_sysexit)
d3561b7f
RR		 925#endif
926
1da177e4 927ENTRY(overflow)
fe7cacc1 928 RING0_INT_FRAME
1da177e4 929 pushl $0
fe7cacc1 930 CFI_ADJUST_CFA_OFFSET 4
1da177e4 931 pushl $do_overflow
fe7cacc1 932 CFI_ADJUST_CFA_OFFSET 4
1da177e4 933 jmp error_code
fe7cacc1 934 CFI_ENDPROC
47a55cd7 935END(overflow)
1da177e4
LT		 936
937ENTRY(bounds)
fe7cacc1 938 RING0_INT_FRAME
1da177e4 939 pushl $0
fe7cacc1 940 CFI_ADJUST_CFA_OFFSET 4
1da177e4 941 pushl $do_bounds
fe7cacc1 942 CFI_ADJUST_CFA_OFFSET 4
1da177e4 943 jmp error_code
fe7cacc1 944 CFI_ENDPROC
47a55cd7 945END(bounds)
1da177e4
LT		 946
947ENTRY(invalid_op)
fe7cacc1 948 RING0_INT_FRAME
1da177e4 949 pushl $0
fe7cacc1 950 CFI_ADJUST_CFA_OFFSET 4
1da177e4 951 pushl $do_invalid_op
fe7cacc1 952 CFI_ADJUST_CFA_OFFSET 4
1da177e4 953 jmp error_code
fe7cacc1 954 CFI_ENDPROC
47a55cd7 955END(invalid_op)
1da177e4
LT		 956
957ENTRY(coprocessor_segment_overrun)
fe7cacc1 958 RING0_INT_FRAME
1da177e4 959 pushl $0
fe7cacc1 960 CFI_ADJUST_CFA_OFFSET 4
1da177e4 961 pushl $do_coprocessor_segment_overrun
fe7cacc1 962 CFI_ADJUST_CFA_OFFSET 4
1da177e4 963 jmp error_code
fe7cacc1 964 CFI_ENDPROC
47a55cd7 965END(coprocessor_segment_overrun)
1da177e4
LT		 966
967ENTRY(invalid_TSS)
fe7cacc1 968 RING0_EC_FRAME
1da177e4 969 pushl $do_invalid_TSS
fe7cacc1 970 CFI_ADJUST_CFA_OFFSET 4
1da177e4 971 jmp error_code
fe7cacc1 972 CFI_ENDPROC
47a55cd7 973END(invalid_TSS)
1da177e4
LT		 974
975ENTRY(segment_not_present)
fe7cacc1 976 RING0_EC_FRAME
1da177e4 977 pushl $do_segment_not_present
fe7cacc1 978 CFI_ADJUST_CFA_OFFSET 4
1da177e4 979 jmp error_code
fe7cacc1 980 CFI_ENDPROC
47a55cd7 981END(segment_not_present)
1da177e4
LT		 982
983ENTRY(stack_segment)
fe7cacc1 984 RING0_EC_FRAME
1da177e4 985 pushl $do_stack_segment
fe7cacc1 986 CFI_ADJUST_CFA_OFFSET 4
1da177e4 987 jmp error_code
fe7cacc1 988 CFI_ENDPROC
47a55cd7 989END(stack_segment)
1da177e4 990
1da177e4 991ENTRY(alignment_check)
fe7cacc1 992 RING0_EC_FRAME
1da177e4 993 pushl $do_alignment_check
fe7cacc1 994 CFI_ADJUST_CFA_OFFSET 4
1da177e4 995 jmp error_code
fe7cacc1 996 CFI_ENDPROC
47a55cd7 997END(alignment_check)
1da177e4 998
d28c4393
P		 999ENTRY(divide_error)
1000 RING0_INT_FRAME
1001 pushl $0 # no error code
1002 CFI_ADJUST_CFA_OFFSET 4
1003 pushl $do_divide_error
fe7cacc1 1004 CFI_ADJUST_CFA_OFFSET 4
1da177e4 1005 jmp error_code
fe7cacc1 1006 CFI_ENDPROC
47a55cd7 1007END(divide_error)
1da177e4
LT		 1008
1009#ifdef CONFIG_X86_MCE
1010ENTRY(machine_check)
fe7cacc1 1011 RING0_INT_FRAME
1da177e4 1012 pushl $0
fe7cacc1 1013 CFI_ADJUST_CFA_OFFSET 4
d2f6f7ae 1014 pushl machine_check_vector
fe7cacc1 1015 CFI_ADJUST_CFA_OFFSET 4
1da177e4 1016 jmp error_code
fe7cacc1 1017 CFI_ENDPROC
47a55cd7 1018END(machine_check)
1da177e4
LT		 1019#endif
1020
1021ENTRY(spurious_interrupt_bug)
fe7cacc1 1022 RING0_INT_FRAME
1da177e4 1023 pushl $0
fe7cacc1 1024 CFI_ADJUST_CFA_OFFSET 4
1da177e4 1025 pushl $do_spurious_interrupt_bug
fe7cacc1 1026 CFI_ADJUST_CFA_OFFSET 4
1da177e4 1027 jmp error_code
fe7cacc1 1028 CFI_ENDPROC
47a55cd7 1029END(spurious_interrupt_bug)
a00e817f
MH		 1030/*
1031 * End of kprobes section
1032 */
1033 .popsection
1da177e4 1034
02ba1a32
AK		 1035ENTRY(kernel_thread_helper)
1036 pushl $0 # fake return address for unwinder
1037 CFI_STARTPROC
1038 movl %edx,%eax
1039 push %edx
1040 CFI_ADJUST_CFA_OFFSET 4
1041 call *%ebx
1042 push %eax
1043 CFI_ADJUST_CFA_OFFSET 4
1044 call do_exit
5f5db591 1045 ud2 # padding for call trace
02ba1a32
AK		 1046 CFI_ENDPROC
1047ENDPROC(kernel_thread_helper)
1048
5ead97c8 1049#ifdef CONFIG_XEN
e2a81baf
JF		 1050/* Xen doesn't set %esp to be precisely what the normal sysenter
1051 entrypoint expects, so fix it up before using the normal path. */
1052ENTRY(xen_sysenter_target)
1053 RING0_INT_FRAME
1054 addl $5*4, %esp /* remove xen-provided frame */
2ddf9b7b 1055 CFI_ADJUST_CFA_OFFSET -5*4
e2a81baf 1056 jmp sysenter_past_esp
557d7d4e 1057 CFI_ENDPROC
e2a81baf 1058
5ead97c8
JF		 1059ENTRY(xen_hypervisor_callback)
1060 CFI_STARTPROC
1061 pushl $0
1062 CFI_ADJUST_CFA_OFFSET 4
1063 SAVE_ALL
1064 TRACE_IRQS_OFF
9ec2b804
JF		 1065
1066 /* Check to see if we got the event in the critical
1067 region in xen_iret_direct, after we've reenabled
1068 events and checked for pending events. This simulates
1069 iret instruction's behaviour where it delivers a
1070 pending interrupt when enabling interrupts. */
1071 movl PT_EIP(%esp),%eax
1072 cmpl $xen_iret_start_crit,%eax
1073 jb 1f
1074 cmpl $xen_iret_end_crit,%eax
1075 jae 1f
1076
0f2c8769 1077 jmp xen_iret_crit_fixup
e2a81baf 1078
e2a81baf 1079ENTRY(xen_do_upcall)
b77797fb 10801: mov %esp, %eax
5ead97c8
JF		 1081 call xen_evtchn_do_upcall
1082 jmp ret_from_intr
1083 CFI_ENDPROC
1084ENDPROC(xen_hypervisor_callback)
1085
1086# Hypervisor uses this for application faults while it executes.
1087# We get here for two reasons:
1088# 1. Fault while reloading DS, ES, FS or GS
1089# 2. Fault while executing IRET
1090# Category 1 we fix up by reattempting the load, and zeroing the segment
1091# register if the load fails.
1092# Category 2 we fix up by jumping to do_iret_error. We cannot use the
1093# normal Linux return path in this case because if we use the IRET hypercall
1094# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1095# We distinguish between categories by maintaining a status value in EAX.
1096ENTRY(xen_failsafe_callback)
1097 CFI_STARTPROC
1098 pushl %eax
1099 CFI_ADJUST_CFA_OFFSET 4
1100 movl $1,%eax
11011: mov 4(%esp),%ds
11022: mov 8(%esp),%es
11033: mov 12(%esp),%fs
11044: mov 16(%esp),%gs
1105 testl %eax,%eax
1106 popl %eax
1107 CFI_ADJUST_CFA_OFFSET -4
1108 lea 16(%esp),%esp
1109 CFI_ADJUST_CFA_OFFSET -16
1110 jz 5f
1111 addl $16,%esp
1112 jmp iret_exc # EAX != 0 => Category 2 (Bad IRET)
11135: pushl $0 # EAX == 0 => Category 1 (Bad segment)
1114 CFI_ADJUST_CFA_OFFSET 4
1115 SAVE_ALL
1116 jmp ret_from_exception
1117 CFI_ENDPROC
1118
1119.section .fixup,"ax"
11206: xorl %eax,%eax
1121 movl %eax,4(%esp)
1122 jmp 1b
11237: xorl %eax,%eax
1124 movl %eax,8(%esp)
1125 jmp 2b
11268: xorl %eax,%eax
1127 movl %eax,12(%esp)
1128 jmp 3b
11299: xorl %eax,%eax
1130 movl %eax,16(%esp)
1131 jmp 4b
1132.previous
1133.section __ex_table,"a"
1134 .align 4
1135 .long 1b,6b
1136 .long 2b,7b
1137 .long 3b,8b
1138 .long 4b,9b
1139.previous
1140ENDPROC(xen_failsafe_callback)
1141
1142#endif /* CONFIG_XEN */
1143
606576ce 1144#ifdef CONFIG_FUNCTION_TRACER
d61f82d0
SR		 1145#ifdef CONFIG_DYNAMIC_FTRACE
1146
1147ENTRY(mcount)
d61f82d0
SR		 1148 ret
1149END(mcount)
1150
1151ENTRY(ftrace_caller)
60a7ecf4
SR		 1152 cmpl $0, function_trace_stop
1153 jne ftrace_stub
1154
d61f82d0
SR		 1155 pushl %eax
1156 pushl %ecx
1157 pushl %edx
1158 movl 0xc(%esp), %eax
1159 movl 0x4(%ebp), %edx
395a59d0 1160 subl $MCOUNT_INSN_SIZE, %eax
d61f82d0
SR		 1161
1162.globl ftrace_call
1163ftrace_call:
1164 call ftrace_stub
1165
1166 popl %edx
1167 popl %ecx
1168 popl %eax
5a45cfe1
SR		 1169#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1170.globl ftrace_graph_call
1171ftrace_graph_call:
1172 jmp ftrace_stub
1173#endif
d61f82d0
SR		 1174
1175.globl ftrace_stub
1176ftrace_stub:
1177 ret
1178END(ftrace_caller)
1179
1180#else /* ! CONFIG_DYNAMIC_FTRACE */
1181
16444a8a 1182ENTRY(mcount)
60a7ecf4
SR		 1183 cmpl $0, function_trace_stop
1184 jne ftrace_stub
1185
16444a8a
ACM		 1186 cmpl $ftrace_stub, ftrace_trace_function
1187 jnz trace
fb52607a 1188#ifdef CONFIG_FUNCTION_GRAPH_TRACER
c2324b69 1189 cmpl $ftrace_stub, ftrace_graph_return
fb52607a 1190 jnz ftrace_graph_caller
e49dc19c
SR		 1191
1192 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
1193 jnz ftrace_graph_caller
caf4b323 1194#endif
16444a8a
ACM		 1195.globl ftrace_stub
1196ftrace_stub:
1197 ret
1198
1199 /* taken from glibc */
1200trace:
1201 pushl %eax
1202 pushl %ecx
1203 pushl %edx
1204 movl 0xc(%esp), %eax
1205 movl 0x4(%ebp), %edx
395a59d0 1206 subl $MCOUNT_INSN_SIZE, %eax
16444a8a 1207
d61f82d0 1208 call *ftrace_trace_function
16444a8a
ACM		 1209
1210 popl %edx
1211 popl %ecx
1212 popl %eax
16444a8a
ACM		 1213 jmp ftrace_stub
1214END(mcount)
d61f82d0 1215#endif /* CONFIG_DYNAMIC_FTRACE */
606576ce 1216#endif /* CONFIG_FUNCTION_TRACER */
16444a8a 1217
fb52607a
FW		 1218#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1219ENTRY(ftrace_graph_caller)
e7d3737e
FW		 1220 cmpl $0, function_trace_stop
1221 jne ftrace_stub
1222
caf4b323
FW		 1223 pushl %eax
1224 pushl %ecx
1225 pushl %edx
1dc1c6ad 1226 movl 0xc(%esp), %edx
caf4b323 1227 lea 0x4(%ebp), %eax
71e308a2 1228 movl (%ebp), %ecx
bb4304c7 1229 subl $MCOUNT_INSN_SIZE, %edx
caf4b323 1230 call prepare_ftrace_return
caf4b323
FW		 1231 popl %edx
1232 popl %ecx
1233 popl %eax
e7d3737e 1234 ret
fb52607a 1235END(ftrace_graph_caller)
caf4b323
FW		 1236
1237.globl return_to_handler
1238return_to_handler:
caf4b323 1239 pushl %eax
caf4b323 1240 pushl %edx
71e308a2 1241 movl %ebp, %eax
caf4b323 1242 call ftrace_return_to_handler
194ec341 1243 movl %eax, %ecx
caf4b323 1244 popl %edx
caf4b323 1245 popl %eax
194ec341 1246 jmp *%ecx
e7d3737e 1247#endif
16444a8a 1248
bb152f53 1249.section .rodata,"a"
541054d9 1250#include "syscall_table_32.S"
1da177e4
LT		 1251
1252syscall_table_size=(.-sys_call_table)
d211af05
AH		 1253
1254/*
1255 * Some functions should be protected against kprobes
1256 */
1257 .pushsection .kprobes.text, "ax"
1258
1259ENTRY(page_fault)
1260 RING0_EC_FRAME
1261 pushl $do_page_fault
1262 CFI_ADJUST_CFA_OFFSET 4
1263 ALIGN
1264error_code:
ccbeed3a
TH		 1265 /* the function address is in %gs's slot on the stack */
1266 pushl %fs
1267 CFI_ADJUST_CFA_OFFSET 4
1268 /*CFI_REL_OFFSET fs, 0*/
d211af05
AH		 1269 pushl %es
1270 CFI_ADJUST_CFA_OFFSET 4
1271 /*CFI_REL_OFFSET es, 0*/
1272 pushl %ds
1273 CFI_ADJUST_CFA_OFFSET 4
1274 /*CFI_REL_OFFSET ds, 0*/
1275 pushl %eax
1276 CFI_ADJUST_CFA_OFFSET 4
1277 CFI_REL_OFFSET eax, 0
1278 pushl %ebp
1279 CFI_ADJUST_CFA_OFFSET 4
1280 CFI_REL_OFFSET ebp, 0
1281 pushl %edi
1282 CFI_ADJUST_CFA_OFFSET 4
1283 CFI_REL_OFFSET edi, 0
1284 pushl %esi
1285 CFI_ADJUST_CFA_OFFSET 4
1286 CFI_REL_OFFSET esi, 0
1287 pushl %edx
1288 CFI_ADJUST_CFA_OFFSET 4
1289 CFI_REL_OFFSET edx, 0
1290 pushl %ecx
1291 CFI_ADJUST_CFA_OFFSET 4
1292 CFI_REL_OFFSET ecx, 0
1293 pushl %ebx
1294 CFI_ADJUST_CFA_OFFSET 4
1295 CFI_REL_OFFSET ebx, 0
1296 cld
d211af05
AH		 1297 movl $(__KERNEL_PERCPU), %ecx
1298 movl %ecx, %fs
1299 UNWIND_ESPFIX_STACK
ccbeed3a
TH		 1300 GS_TO_REG %ecx
1301 movl PT_GS(%esp), %edi # get the function address
d211af05
AH		 1302 movl PT_ORIG_EAX(%esp), %edx # get the error code
1303 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
ccbeed3a
TH		 1304 REG_TO_PTGS %ecx
1305 SET_KERNEL_GS %ecx
d211af05
AH		 1306 movl $(__USER_DS), %ecx
1307 movl %ecx, %ds
1308 movl %ecx, %es
1309 TRACE_IRQS_OFF
1310 movl %esp,%eax # pt_regs pointer
1311 call *%edi
1312 jmp ret_from_exception
1313 CFI_ENDPROC
1314END(page_fault)
1315
1316/*
1317 * Debug traps and NMI can happen at the one SYSENTER instruction
1318 * that sets up the real kernel stack. Check here, since we can't
1319 * allow the wrong stack to be used.
1320 *
1321 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
1322 * already pushed 3 words if it hits on the sysenter instruction:
1323 * eflags, cs and eip.
1324 *
1325 * We just load the right stack, and push the three (known) values
1326 * by hand onto the new stack - while updating the return eip past
1327 * the instruction that would have done it for sysenter.
1328 */
f0d96110
TH		 1329.macro FIX_STACK offset ok label
1330 cmpw $__KERNEL_CS, 4(%esp)
1331 jne \ok
1332\label:
1333 movl TSS_sysenter_sp0 + \offset(%esp), %esp
1334 CFI_DEF_CFA esp, 0
1335 CFI_UNDEFINED eip
1336 pushfl
1337 CFI_ADJUST_CFA_OFFSET 4
1338 pushl $__KERNEL_CS
1339 CFI_ADJUST_CFA_OFFSET 4
1340 pushl $sysenter_past_esp
1341 CFI_ADJUST_CFA_OFFSET 4
d211af05 1342 CFI_REL_OFFSET eip, 0
f0d96110 1343.endm
d211af05
AH		 1344
1345ENTRY(debug)
1346 RING0_INT_FRAME
1347 cmpl $ia32_sysenter_target,(%esp)
1348 jne debug_stack_correct
f0d96110 1349 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
d211af05
AH		 1350debug_stack_correct:
1351 pushl $-1 # mark this as an int
1352 CFI_ADJUST_CFA_OFFSET 4
1353 SAVE_ALL
1354 TRACE_IRQS_OFF
1355 xorl %edx,%edx # error code 0
1356 movl %esp,%eax # pt_regs pointer
1357 call do_debug
1358 jmp ret_from_exception
1359 CFI_ENDPROC
1360END(debug)
1361
1362/*
1363 * NMI is doubly nasty. It can happen _while_ we're handling
1364 * a debug fault, and the debug fault hasn't yet been able to
1365 * clear up the stack. So we first check whether we got an
1366 * NMI on the sysenter entry path, but after that we need to
1367 * check whether we got an NMI on the debug path where the debug
1368 * fault happened on the sysenter path.
1369 */
1370ENTRY(nmi)
1371 RING0_INT_FRAME
1372 pushl %eax
1373 CFI_ADJUST_CFA_OFFSET 4
1374 movl %ss, %eax
1375 cmpw $__ESPFIX_SS, %ax
1376 popl %eax
1377 CFI_ADJUST_CFA_OFFSET -4
1378 je nmi_espfix_stack
1379 cmpl $ia32_sysenter_target,(%esp)
1380 je nmi_stack_fixup
1381 pushl %eax
1382 CFI_ADJUST_CFA_OFFSET 4
1383 movl %esp,%eax
1384 /* Do not access memory above the end of our stack page,
1385 * it might not exist.
1386 */
1387 andl $(THREAD_SIZE-1),%eax
1388 cmpl $(THREAD_SIZE-20),%eax
1389 popl %eax
1390 CFI_ADJUST_CFA_OFFSET -4
1391 jae nmi_stack_correct
1392 cmpl $ia32_sysenter_target,12(%esp)
1393 je nmi_debug_stack_check
1394nmi_stack_correct:
1395 /* We have a RING0_INT_FRAME here */
1396 pushl %eax
1397 CFI_ADJUST_CFA_OFFSET 4
1398 SAVE_ALL
d211af05
AH		 1399 xorl %edx,%edx # zero error code
1400 movl %esp,%eax # pt_regs pointer
1401 call do_nmi
2e04bc76 1402 jmp restore_all_notrace
d211af05
AH		 1403 CFI_ENDPROC
1404
1405nmi_stack_fixup:
1406 RING0_INT_FRAME
f0d96110 1407 FIX_STACK 12, nmi_stack_correct, 1
d211af05
AH		 1408 jmp nmi_stack_correct
1409
1410nmi_debug_stack_check:
1411 /* We have a RING0_INT_FRAME here */
1412 cmpw $__KERNEL_CS,16(%esp)
1413 jne nmi_stack_correct
1414 cmpl $debug,(%esp)
1415 jb nmi_stack_correct
1416 cmpl $debug_esp_fix_insn,(%esp)
1417 ja nmi_stack_correct
f0d96110 1418 FIX_STACK 24, nmi_stack_correct, 1
d211af05
AH		 1419 jmp nmi_stack_correct
1420
1421nmi_espfix_stack:
1422 /* We have a RING0_INT_FRAME here.
1423 *
1424 * create the pointer to lss back
1425 */
1426 pushl %ss
1427 CFI_ADJUST_CFA_OFFSET 4
1428 pushl %esp
1429 CFI_ADJUST_CFA_OFFSET 4
bda3a897 1430 addl $4, (%esp)
d211af05
AH		 1431 /* copy the iret frame of 12 bytes */
1432 .rept 3
1433 pushl 16(%esp)
1434 CFI_ADJUST_CFA_OFFSET 4
1435 .endr
1436 pushl %eax
1437 CFI_ADJUST_CFA_OFFSET 4
1438 SAVE_ALL
d211af05
AH		 1439 FIXUP_ESPFIX_STACK # %eax == %esp
1440 xorl %edx,%edx # zero error code
1441 call do_nmi
1442 RESTORE_REGS
1443 lss 12+4(%esp), %esp # back to espfix stack
1444 CFI_ADJUST_CFA_OFFSET -24
1445 jmp irq_return
1446 CFI_ENDPROC
1447END(nmi)
1448
1449ENTRY(int3)
1450 RING0_INT_FRAME
1451 pushl $-1 # mark this as an int
1452 CFI_ADJUST_CFA_OFFSET 4
1453 SAVE_ALL
1454 TRACE_IRQS_OFF
1455 xorl %edx,%edx # zero error code
1456 movl %esp,%eax # pt_regs pointer
1457 call do_int3
1458 jmp ret_from_exception
1459 CFI_ENDPROC
1460END(int3)
1461
1462ENTRY(general_protection)
1463 RING0_EC_FRAME
1464 pushl $do_general_protection
1465 CFI_ADJUST_CFA_OFFSET 4
1466 jmp error_code
1467 CFI_ENDPROC
1468END(general_protection)
1469
1470/*
1471 * End of kprobes section
1472 */
1473 .popsection
Linux 6.x block layer and io_uring tree(s)