3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
21 #include <linux/errno.h>
22 #include <linux/err.h>
23 #include <linux/magic.h>
24 #include <asm/unistd.h>
25 #include <asm/processor.h>
28 #include <asm/thread_info.h>
29 #include <asm/ppc_asm.h>
30 #include <asm/asm-offsets.h>
31 #include <asm/cputable.h>
32 #include <asm/firmware.h>
34 #include <asm/ptrace.h>
35 #include <asm/irqflags.h>
36 #include <asm/ftrace.h>
37 #include <asm/hw_irq.h>
38 #include <asm/context_tracking.h>
40 #include <asm/ppc-opcode.h>
47 .tc sys_call_table[TC],sys_call_table
49 /* This value is used to mark exception frames on the stack. */
51 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
56 .globl system_call_common
58 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
60 extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
62 END_FTR_SECTION_IFSET(CPU_FTR_TM)
66 addi r1,r1,-INT_FRAME_SIZE
74 beq 2f /* if from kernel mode */
75 ACCOUNT_CPU_USER_ENTRY(r13, r10, r11)
94 * This clears CR0.SO (bit 28), which is the error indication on
95 * return from this system call.
97 rldimi r2,r11,28,(63-28)
104 addi r9,r1,STACK_FRAME_OVERHEAD
105 ld r11,exception_marker@toc(r2)
106 std r11,-16(r9) /* "regshere" marker */
107 #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
110 /* if from user, see if there are any DTL entries to process */
111 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
112 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
113 addi r10,r10,LPPACA_DTLIDX
114 LDX_BE r10,0,r10 /* get log write index */
117 bl accumulate_stolen_time
121 addi r9,r1,STACK_FRAME_OVERHEAD
123 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
124 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
127 * A syscall should always be called with interrupts enabled
128 * so we just unconditionally hard-enable here. When some kind
129 * of irq tracing is used, we additionally check that condition
132 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
133 lbz r10,PACASOFTIRQEN(r13)
136 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
139 #ifdef CONFIG_PPC_BOOK3E
145 #endif /* CONFIG_PPC_BOOK3E */
147 /* We do need to set SOFTE in the stack frame or the return
148 * from interrupt will be painful
153 CURRENT_THREAD_INFO(r11, r1)
155 andi. r11,r10,_TIF_SYSCALL_DOTRACE
156 bne syscall_dotrace /* does not return */
157 cmpldi 0,r0,NR_syscalls
160 system_call: /* label this so stack traces look sane */
162 * Need to vector to 32 Bit or default sys_call_table here,
163 * based on caller's run-mode / personality.
165 ld r11,SYS_CALL_TABLE@toc(2)
166 andi. r10,r10,_TIF_32BIT
168 addi r11,r11,8 /* use 32-bit syscall entries */
177 ldx r12,r11,r0 /* Fetch system call handler [ptr] */
179 bctrl /* Call handler */
183 CURRENT_THREAD_INFO(r12, r1)
186 #ifdef CONFIG_PPC_BOOK3S
187 /* No MSR:RI on BookE */
192 * Disable interrupts so current_thread_info()->flags can't change,
193 * and so that we don't get interrupted after loading SRR0/1.
195 #ifdef CONFIG_PPC_BOOK3E
199 * For performance reasons we clear RI the same time that we
200 * clear EE. We only need to clear RI just before we restore r13
201 * below, but batching it with EE saves us one expensive mtmsrd call.
202 * We have to be careful to restore RI if we branch anywhere from
203 * here (eg syscall_exit_work).
207 #endif /* CONFIG_PPC_BOOK3E */
211 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
212 bne- syscall_exit_work
216 #ifdef CONFIG_ALTIVEC
217 andis. r0,r8,MSR_VEC@h
220 2: addi r3,r1,STACK_FRAME_OVERHEAD
221 #ifdef CONFIG_PPC_BOOK3S
223 mtmsrd r10,1 /* Restore RI */
226 #ifdef CONFIG_PPC_BOOK3S
237 .Lsyscall_error_cont:
240 stdcx. r0,0,r1 /* to clear the reservation */
241 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
246 ACCOUNT_CPU_USER_EXIT(r13, r11, r12)
250 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
252 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
260 b . /* prevent speculative execution */
263 oris r5,r5,0x1000 /* Set SO bit in CR */
266 b .Lsyscall_error_cont
268 /* Traced system call support */
271 addi r3,r1,STACK_FRAME_OVERHEAD
272 bl do_syscall_trace_enter
275 * We use the return value of do_syscall_trace_enter() as the syscall
276 * number. If the syscall was rejected for any reason do_syscall_trace_enter()
277 * returns an invalid syscall number and the test below against
278 * NR_syscalls will fail.
282 /* Restore argument registers just clobbered and/or possibly changed. */
290 /* Repopulate r9 and r10 for the system_call path */
291 addi r9,r1,STACK_FRAME_OVERHEAD
292 CURRENT_THREAD_INFO(r10, r1)
295 cmpldi r0,NR_syscalls
298 /* Return code is already in r3 thanks to do_syscall_trace_enter() */
307 #ifdef CONFIG_PPC_BOOK3S
309 mtmsrd r10,1 /* Restore RI */
311 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
312 If TIF_NOERROR is set, just save r3 as it is. */
314 andi. r0,r9,_TIF_RESTOREALL
318 0: cmpld r3,r11 /* r11 is -MAX_ERRNO */
320 andi. r0,r9,_TIF_NOERROR
324 oris r5,r5,0x1000 /* Set SO bit in CR */
327 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
330 /* Clear per-syscall TIF flags if any are set. */
332 li r11,_TIF_PERSYSCALL_MASK
333 addi r12,r12,TI_FLAGS
338 subi r12,r12,TI_FLAGS
340 4: /* Anything else left to do? */
342 lis r3,INIT_PPR@highest /* Set thread.ppr = 3 */
343 ld r10,PACACURRENT(r13)
344 sldi r3,r3,32 /* bits 11-13 are used for ppr */
345 std r3,TASKTHREADPPR(r10)
346 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
348 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
349 beq ret_from_except_lite
351 /* Re-enable interrupts */
352 #ifdef CONFIG_PPC_BOOK3E
358 #endif /* CONFIG_PPC_BOOK3E */
361 addi r3,r1,STACK_FRAME_OVERHEAD
362 bl do_syscall_trace_leave
365 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
367 /* Firstly we need to enable TM in the kernel */
370 rldimi r10, r9, MSR_TM_LG, 63-MSR_TM_LG
373 /* tabort, this dooms the transaction, nothing else */
374 li r9, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)
378 * Return directly to userspace. We have corrupted user register state,
379 * but userspace will never see that register state. Execution will
380 * resume after the tbegin of the aborted transaction with the
381 * checkpointed register state.
390 b . /* prevent speculative execution */
393 /* Save non-volatile GPRs, if not already saved. */
405 * The sigsuspend and rt_sigsuspend system calls can call do_signal
406 * and thus put the process into the stopped state where we might
407 * want to examine its user state with ptrace. Therefore we need
408 * to save all the nonvolatile registers (r14 - r31) before calling
409 * the C code. Similarly, fork, vfork and clone need the full
410 * register state on the stack so that it can be copied to the child.
428 _GLOBAL(ppc32_swapcontext)
430 bl compat_sys_swapcontext
433 _GLOBAL(ppc64_swapcontext)
438 _GLOBAL(ppc_switch_endian)
443 _GLOBAL(ret_from_fork)
449 _GLOBAL(ret_from_kernel_thread)
454 #ifdef PPC64_ELF_ABI_v2
462 * This routine switches between two different tasks. The process
463 * state of one is saved on its kernel stack. Then the state
464 * of the other is restored from its kernel stack. The memory
465 * management hardware is updated to the second process's state.
466 * Finally, we can return to the second process, via ret_from_except.
467 * On entry, r3 points to the THREAD for the current task, r4
468 * points to the THREAD for the new task.
470 * Note: there are two ways to get to the "going out" portion
471 * of this code; either by coming in via the entry (_switch)
472 * or via "fork" which must set up an environment equivalent
473 * to the "_switch" path. If you change this you'll have to change
474 * the fork code also.
476 * The code which creates the new task context is in 'copy_thread'
477 * in arch/powerpc/kernel/process.c
483 stdu r1,-SWITCH_FRAME_SIZE(r1)
484 /* r3-r13 are caller saved -- Cort */
487 std r0,_NIP(r1) /* Return to switch caller */
490 std r1,KSP(r3) /* Set old stack pointer */
493 /* We need a sync somewhere here to make sure that if the
494 * previous task gets rescheduled on another CPU, it sees all
495 * stores it has performed on this one.
498 #endif /* CONFIG_SMP */
501 * If we optimise away the clear of the reservation in system
502 * calls because we know the CPU tracks the address of the
503 * reservation, then we need to clear it here to cover the
504 * case that the kernel context switch path has no larx
509 END_FTR_SECTION_IFSET(CPU_FTR_STCX_CHECKS_ADDRESS)
513 * A cp_abort (copy paste abort) here ensures that when context switching, a
514 * copy from one process can't leak into the paste of another.
517 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
519 #ifdef CONFIG_PPC_BOOK3S
520 /* Cancel all explict user streams as they will have no use after context
521 * switch and will stop the HW from creating streams itself
523 DCBT_STOP_ALL_STREAM_IDS(r6)
526 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
527 std r6,PACACURRENT(r13) /* Set new 'current' */
529 ld r8,KSP(r4) /* new stack pointer */
530 #ifdef CONFIG_PPC_STD_MMU_64
531 BEGIN_MMU_FTR_SECTION
533 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
535 clrrdi r6,r8,28 /* get its ESID */
536 clrrdi r9,r1,28 /* get current sp ESID */
538 clrrdi r6,r8,40 /* get its 1T ESID */
539 clrrdi r9,r1,40 /* get current sp 1T ESID */
540 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT)
541 clrldi. r0,r6,2 /* is new ESID c00000000? */
542 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
544 beq 2f /* if yes, don't slbie it */
546 /* Bolt in the new stack SLB entry */
547 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
548 oris r0,r6,(SLB_ESID_V)@h
549 ori r0,r0,(SLB_NUM_BOLTED-1)@l
551 li r9,MMU_SEGSIZE_1T /* insert B field */
552 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
553 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
554 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
556 /* Update the last bolted SLB. No write barriers are needed
557 * here, provided we only update the current CPU's SLB shadow
560 ld r9,PACA_SLBSHADOWPTR(r13)
562 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
563 li r12,SLBSHADOW_STACKVSID
564 STDX_BE r7,r12,r9 /* Save VSID */
565 li r12,SLBSHADOW_STACKESID
566 STDX_BE r0,r12,r9 /* Save ESID */
568 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
569 * we have 1TB segments, the only CPUs known to have the errata
570 * only support less than 1TB of system memory and we'll never
571 * actually hit this code path.
575 slbie r6 /* Workaround POWER5 < DD2.1 issue */
579 #endif /* CONFIG_PPC_STD_MMU_64 */
581 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
582 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
583 because we don't need to leave the 288-byte ABI gap at the
584 top of the kernel stack. */
585 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
587 mr r1,r8 /* start using new stack pointer */
588 std r7,PACAKSAVE(r13)
593 /* r3-r13 are destroyed -- Cort */
597 /* convert old thread to its task_struct for return value */
599 ld r7,_NIP(r1) /* Return to _switch caller in new task */
601 addi r1,r1,SWITCH_FRAME_SIZE
605 _GLOBAL(ret_from_except)
608 bne ret_from_except_lite
611 _GLOBAL(ret_from_except_lite)
613 * Disable interrupts so that current_thread_info()->flags
614 * can't change between when we test it and when we return
615 * from the interrupt.
617 #ifdef CONFIG_PPC_BOOK3E
621 mtmsrd r10,1 /* Update machine state */
622 #endif /* CONFIG_PPC_BOOK3E */
624 CURRENT_THREAD_INFO(r9, r1)
626 #ifdef CONFIG_PPC_BOOK3E
627 ld r10,PACACURRENT(r13)
628 #endif /* CONFIG_PPC_BOOK3E */
632 #ifdef CONFIG_PPC_BOOK3E
633 lwz r3,(THREAD+THREAD_DBCR0)(r10)
634 #endif /* CONFIG_PPC_BOOK3E */
636 /* Check current_thread_info()->flags */
637 andi. r0,r4,_TIF_USER_WORK_MASK
639 #ifdef CONFIG_PPC_BOOK3E
641 * Check to see if the dbcr0 register is set up to debug.
642 * Use the internal debug mode bit to do this.
644 andis. r0,r3,DBCR0_IDM@h
647 rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
654 addi r3,r1,STACK_FRAME_OVERHEAD
658 1: andi. r0,r4,_TIF_NEED_RESCHED
660 bl restore_interrupts
662 b ret_from_except_lite
664 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
665 andi. r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
666 bne 3f /* only restore TM if nothing else to do */
667 addi r3,r1,STACK_FRAME_OVERHEAD
674 * Use a non volatile GPR to save and restore our thread_info flags
675 * across the call to restore_interrupts.
678 bl restore_interrupts
680 addi r3,r1,STACK_FRAME_OVERHEAD
685 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
686 andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
689 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
692 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
693 mr r4,r1 /* src: current exception frame */
694 mr r1,r3 /* Reroute the trampoline frame to r1 */
696 /* Copy from the original to the trampoline. */
697 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
698 li r6,0 /* start offset: 0 */
705 /* Do real store operation to complete stwu */
709 /* Clear _TIF_EMULATE_STACK_STORE flag */
710 lis r11,_TIF_EMULATE_STACK_STORE@h
718 #ifdef CONFIG_PREEMPT
719 /* Check if we need to preempt */
720 andi. r0,r4,_TIF_NEED_RESCHED
722 /* Check that preempt_count() == 0 and interrupts are enabled */
723 lwz r8,TI_PREEMPT(r9)
727 crandc eq,cr1*4+eq,eq
731 * Here we are preempting the current task. We want to make
732 * sure we are soft-disabled first and reconcile irq state.
734 RECONCILE_IRQ_STATE(r3,r4)
735 1: bl preempt_schedule_irq
737 /* Re-test flags and eventually loop */
738 CURRENT_THREAD_INFO(r9, r1)
740 andi. r0,r4,_TIF_NEED_RESCHED
744 * arch_local_irq_restore() from preempt_schedule_irq above may
745 * enable hard interrupt but we really should disable interrupts
746 * when we return from the interrupt, and so that we don't get
747 * interrupted after loading SRR0/1.
749 #ifdef CONFIG_PPC_BOOK3E
753 mtmsrd r10,1 /* Update machine state */
754 #endif /* CONFIG_PPC_BOOK3E */
755 #endif /* CONFIG_PREEMPT */
757 .globl fast_exc_return_irq
761 * This is the main kernel exit path. First we check if we
762 * are about to re-enable interrupts
765 lbz r6,PACASOFTIRQEN(r13)
769 /* We are enabling, were we already enabled ? Yes, just return */
774 * We are about to soft-enable interrupts (we are hard disabled
775 * at this point). We check if there's anything that needs to
778 lbz r0,PACAIRQHAPPENED(r13)
780 bne- restore_check_irq_replay
783 * Get here when nothing happened while soft-disabled, just
784 * soft-enable and move-on. We will hard-enable as a side
790 stb r0,PACASOFTIRQEN(r13);
793 * Final return path. BookE is handled in a different file
796 #ifdef CONFIG_PPC_BOOK3E
797 b exception_return_book3e
800 * Clear the reservation. If we know the CPU tracks the address of
801 * the reservation then we can potentially save some cycles and use
802 * a larx. On POWER6 and POWER7 this is significantly faster.
805 stdcx. r0,0,r1 /* to clear the reservation */
808 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
811 * Some code path such as load_up_fpu or altivec return directly
812 * here. They run entirely hard disabled and do not alter the
813 * interrupt state. They also don't use lwarx/stwcx. and thus
814 * are known not to leave dangling reservations.
816 .globl fast_exception_return
817 fast_exception_return:
831 /* Load PPR from thread struct before we clear MSR:RI */
833 ld r2,PACACURRENT(r13)
834 ld r2,TASKTHREADPPR(r2)
835 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
838 * Clear RI before restoring r13. If we are returning to
839 * userspace and we take an exception after restoring r13,
840 * we end up corrupting the userspace r13 value.
845 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
847 std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
850 * r13 is our per cpu area, only restore it if we are returning to
851 * userspace the value stored in the stack frame may belong to
857 mtspr SPRN_PPR,r2 /* Restore PPR */
858 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
859 ACCOUNT_CPU_USER_EXIT(r13, r2, r4)
876 b . /* prevent speculative execution */
878 #endif /* CONFIG_PPC_BOOK3E */
881 * We are returning to a context with interrupts soft disabled.
883 * However, we may also about to hard enable, so we need to
884 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
885 * or that bit can get out of sync and bad things will happen
889 lbz r7,PACAIRQHAPPENED(r13)
892 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
893 stb r7,PACAIRQHAPPENED(r13)
895 stb r0,PACASOFTIRQEN(r13);
900 * Something did happen, check if a re-emit is needed
901 * (this also clears paca->irq_happened)
903 restore_check_irq_replay:
904 /* XXX: We could implement a fast path here where we check
905 * for irq_happened being just 0x01, in which case we can
906 * clear it and return. That means that we would potentially
907 * miss a decrementer having wrapped all the way around.
909 * Still, this might be useful for things like hash_page
911 bl __check_irq_replay
913 beq restore_no_replay
916 * We need to re-emit an interrupt. We do so by re-using our
917 * existing exception frame. We first change the trap value,
918 * but we need to ensure we preserve the low nibble of it
926 * Then find the right handler and call it. Interrupts are
927 * still soft-disabled and we keep them that way.
931 addi r3,r1,STACK_FRAME_OVERHEAD;
934 1: cmpwi cr0,r3,0xe60
936 addi r3,r1,STACK_FRAME_OVERHEAD;
937 bl handle_hmi_exception
939 1: cmpwi cr0,r3,0x900
941 addi r3,r1,STACK_FRAME_OVERHEAD;
944 #ifdef CONFIG_PPC_DOORBELL
946 #ifdef CONFIG_PPC_BOOK3E
953 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
954 #endif /* CONFIG_PPC_BOOK3E */
956 addi r3,r1,STACK_FRAME_OVERHEAD;
957 bl doorbell_exception
959 #endif /* CONFIG_PPC_DOORBELL */
960 1: b ret_from_except /* What else to do here ? */
963 addi r3,r1,STACK_FRAME_OVERHEAD
964 bl unrecoverable_exception
967 #ifdef CONFIG_PPC_RTAS
969 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
970 * called with the MMU off.
972 * In addition, we need to be in 32b mode, at least for now.
974 * Note: r3 is an input parameter to rtas, so don't trash it...
979 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
981 /* Because RTAS is running in 32b mode, it clobbers the high order half
982 * of all registers that it saves. We therefore save those registers
983 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
985 SAVE_GPR(2, r1) /* Save the TOC */
986 SAVE_GPR(13, r1) /* Save paca */
987 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
988 SAVE_10GPRS(22, r1) /* ditto */
1001 /* Temporary workaround to clear CR until RTAS can be modified to
1008 /* There is no way it is acceptable to get here with interrupts enabled,
1009 * check it with the asm equivalent of WARN_ON
1011 lbz r0,PACASOFTIRQEN(r13)
1013 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1016 /* Hard-disable interrupts */
1022 /* Unfortunately, the stack pointer and the MSR are also clobbered,
1023 * so they are saved in the PACA which allows us to restore
1024 * our original state after RTAS returns.
1027 std r6,PACASAVEDMSR(r13)
1029 /* Setup our real return addr */
1030 LOAD_REG_ADDR(r4,rtas_return_loc)
1031 clrldi r4,r4,2 /* convert to realmode address */
1035 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
1039 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
1040 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI|MSR_LE
1042 sync /* disable interrupts so SRR0/1 */
1043 mtmsrd r0 /* don't get trashed */
1045 LOAD_REG_ADDR(r4, rtas)
1046 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
1047 ld r4,RTASBASE(r4) /* get the rtas->base value */
1052 b . /* prevent speculative execution */
1057 /* relocation is off at this point */
1059 clrldi r4,r4,2 /* convert to realmode address */
1063 ld r3,(1f-0b)(r3) /* get &rtas_restore_regs */
1071 ld r1,PACAR1(r4) /* Restore our SP */
1072 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
1077 b . /* prevent speculative execution */
1080 1: .llong rtas_restore_regs
1083 /* relocation is on at this point */
1084 REST_GPR(2, r1) /* Restore the TOC */
1085 REST_GPR(13, r1) /* Restore paca */
1086 REST_8GPRS(14, r1) /* Restore the non-volatiles */
1087 REST_10GPRS(22, r1) /* ditto */
1102 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
1103 ld r0,16(r1) /* get return address */
1106 blr /* return to caller */
1108 #endif /* CONFIG_PPC_RTAS */
1113 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1115 /* Because PROM is running in 32b mode, it clobbers the high order half
1116 * of all registers that it saves. We therefore save those registers
1117 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1128 /* Put PROM address in SRR0 */
1131 /* Setup our trampoline return addr in LR */
1134 addi r4,r4,(1f - 0b)
1137 /* Prepare a 32-bit mode big endian MSR
1139 #ifdef CONFIG_PPC_BOOK3E
1140 rlwinm r11,r11,0,1,31
1143 #else /* CONFIG_PPC_BOOK3E */
1144 LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_ISF | MSR_LE)
1148 #endif /* CONFIG_PPC_BOOK3E */
1150 1: /* Return from OF */
1153 /* Just make sure that r1 top 32 bits didn't get
1158 /* Restore the MSR (back to 64 bits) */
1163 /* Restore other registers */
1171 addi r1,r1,PROM_FRAME_SIZE
1176 #ifdef CONFIG_FUNCTION_TRACER
1177 #ifdef CONFIG_DYNAMIC_FTRACE
1185 #ifndef CC_USING_MPROFILE_KERNEL
1186 _GLOBAL_TOC(ftrace_caller)
1187 /* Taken from output of objdump from lib64/glibc */
1193 subi r3, r3, MCOUNT_INSN_SIZE
1198 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1199 .globl ftrace_graph_call
1202 _GLOBAL(ftrace_graph_stub)
1208 #else /* CC_USING_MPROFILE_KERNEL */
1211 * ftrace_caller() is the function that replaces _mcount() when ftrace is
1214 * We arrive here after a function A calls function B, and we are the trace
1215 * function for B. When we enter r1 points to A's stack frame, B has not yet
1216 * had a chance to allocate one yet.
1218 * Additionally r2 may point either to the TOC for A, or B, depending on
1219 * whether B did a TOC setup sequence before calling us.
1221 * On entry the LR points back to the _mcount() call site, and r0 holds the
1222 * saved LR as it was on entry to B, ie. the original return address at the
1225 * Our job is to save the register state into a struct pt_regs (on the stack)
1226 * and then arrange for the ftrace function to be called.
1228 _GLOBAL(ftrace_caller)
1229 /* Save the original return address in A's stack frame */
1232 /* Create our stack frame + pt_regs */
1233 stdu r1,-SWITCH_FRAME_SIZE(r1)
1235 /* Save all gprs to pt_regs */
1241 /* Load special regs for save below */
1247 /* Get the _mcount() call site out of LR */
1249 /* Save it as pt_regs->nip & pt_regs->link */
1253 /* Save callee's TOC in the ABI compliant location */
1255 ld r2,PACATOC(r13) /* get kernel TOC in r2 */
1257 addis r3,r2,function_trace_op@toc@ha
1258 addi r3,r3,function_trace_op@toc@l
1261 #ifdef CONFIG_LIVEPATCH
1262 mr r14,r7 /* remember old NIP */
1264 /* Calculate ip from nip-4 into r3 for call below */
1265 subi r3, r7, MCOUNT_INSN_SIZE
1267 /* Put the original return address in r4 as parent_ip */
1270 /* Save special regs */
1276 /* Load &pt_regs in r6 for call below */
1277 addi r6, r1 ,STACK_FRAME_OVERHEAD
1279 /* ftrace_call(r3, r4, r5, r6) */
1285 /* Load ctr with the possibly modified NIP */
1288 #ifdef CONFIG_LIVEPATCH
1289 cmpd r14,r3 /* has NIP been altered? */
1298 /* Restore callee's TOC */
1301 /* Pop our stack frame */
1302 addi r1, r1, SWITCH_FRAME_SIZE
1304 /* Restore original LR for return to B */
1308 #ifdef CONFIG_LIVEPATCH
1309 /* Based on the cmpd above, if the NIP was altered handle livepatch */
1310 bne- livepatch_handler
1313 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1315 .globl ftrace_graph_call
1318 _GLOBAL(ftrace_graph_stub)
1322 ld r0,LRSAVE(r1) /* restore callee's lr at _mcount site */
1324 bctr /* jump after _mcount site */
1325 #endif /* CC_USING_MPROFILE_KERNEL */
1327 _GLOBAL(ftrace_stub)
1330 #ifdef CONFIG_LIVEPATCH
1332 * This function runs in the mcount context, between two functions. As
1333 * such it can only clobber registers which are volatile and used in
1336 * We get here when a function A, calls another function B, but B has
1337 * been live patched with a new function C.
1340 * - we have no stack frame and can not allocate one
1341 * - LR points back to the original caller (in A)
1342 * - CTR holds the new NIP in C
1343 * - r0 & r12 are free
1345 * r0 can't be used as the base register for a DS-form load or store, so
1346 * we temporarily shuffle r1 (stack pointer) into r0 and then put it back.
1349 CURRENT_THREAD_INFO(r12, r1)
1351 /* Save stack pointer into r0 */
1354 /* Allocate 3 x 8 bytes */
1355 ld r1, TI_livepatch_sp(r12)
1357 std r1, TI_livepatch_sp(r12)
1359 /* Save toc & real LR on livepatch stack */
1364 /* Store stack end marker */
1365 lis r12, STACK_END_MAGIC@h
1366 ori r12, r12, STACK_END_MAGIC@l
1369 /* Restore real stack pointer */
1372 /* Put ctr in r12 for global entry and branch there */
1377 * Now we are returning from the patched function to the original
1378 * caller A. We are free to use r0 and r12, and we can use r2 until we
1382 CURRENT_THREAD_INFO(r12, r1)
1384 /* Save stack pointer into r0 */
1387 ld r1, TI_livepatch_sp(r12)
1389 /* Check stack marker hasn't been trashed */
1390 lis r2, STACK_END_MAGIC@h
1391 ori r2, r2, STACK_END_MAGIC@l
1394 EMIT_BUG_ENTRY 1b, __FILE__, __LINE__ - 1, 0
1396 /* Restore LR & toc from livepatch stack */
1401 /* Pop livepatch stack frame */
1402 CURRENT_THREAD_INFO(r12, r0)
1404 std r1, TI_livepatch_sp(r12)
1406 /* Restore real stack pointer */
1409 /* Return to original caller of live patched function */
1415 _GLOBAL_TOC(_mcount)
1416 /* Taken from output of objdump from lib64/glibc */
1423 subi r3, r3, MCOUNT_INSN_SIZE
1424 LOAD_REG_ADDR(r5,ftrace_trace_function)
1432 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1433 b ftrace_graph_caller
1438 _GLOBAL(ftrace_stub)
1441 #endif /* CONFIG_DYNAMIC_FTRACE */
1443 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1444 #ifndef CC_USING_MPROFILE_KERNEL
1445 _GLOBAL(ftrace_graph_caller)
1446 /* load r4 with local address */
1448 subi r4, r4, MCOUNT_INSN_SIZE
1450 /* Grab the LR out of the caller stack frame */
1454 bl prepare_ftrace_return
1458 * prepare_ftrace_return gives us the address we divert to.
1459 * Change the LR in the callers stack frame to this.
1469 #else /* CC_USING_MPROFILE_KERNEL */
1470 _GLOBAL(ftrace_graph_caller)
1471 /* with -mprofile-kernel, parameter regs are still alive at _mcount */
1481 /* Save callee's TOC in the ABI compliant location */
1483 ld r2, PACATOC(r13) /* get kernel TOC in r2 */
1485 mfctr r4 /* ftrace_caller has moved local addr here */
1487 mflr r3 /* ftrace_caller has restored LR from stack */
1488 subi r4, r4, MCOUNT_INSN_SIZE
1490 bl prepare_ftrace_return
1494 * prepare_ftrace_return gives us the address we divert to.
1495 * Change the LR to this.
1510 /* Restore callee's TOC */
1517 #endif /* CC_USING_MPROFILE_KERNEL */
1519 _GLOBAL(return_to_handler)
1520 /* need to save return values */
1530 * We might be called from a module.
1531 * Switch to our TOC to run inside the core kernel.
1535 bl ftrace_return_to_handler
1538 /* return value has real return address */
1547 /* Jump back to real return address */
1549 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1550 #endif /* CONFIG_FUNCTION_TRACER */