[linux-2.6-block.git] / arch / powerpc / kernel / tm.S

/*
 * Transactional memory support routines to reclaim and recheckpoint
 * transactional process state.
 *
 * Copyright 2012 Matt Evans & Michael Neuling, IBM Corporation.
 */

#include <asm/asm-offsets.h>
#include <asm/ppc_asm.h>
#include <asm/ppc-opcode.h>
#include <asm/ptrace.h>
#include <asm/reg.h>
#include <asm/bug.h>

#ifdef CONFIG_VSX
/* See fpu.S, this is borrowed from there */
#define __SAVE_32FPRS_VSRS(n,c,base)		\
BEGIN_FTR_SECTION				\
	b	2f;				\
END_FTR_SECTION_IFSET(CPU_FTR_VSX);		\
	SAVE_32FPRS(n,base);			\
	b	3f;				\
2:	SAVE_32VSRS(n,c,base);			\
3:
#define __REST_32FPRS_VSRS(n,c,base)		\
BEGIN_FTR_SECTION				\
	b	2f;				\
END_FTR_SECTION_IFSET(CPU_FTR_VSX);		\
	REST_32FPRS(n,base);			\
	b	3f;				\
2:	REST_32VSRS(n,c,base);			\
3:
#else
#define __SAVE_32FPRS_VSRS(n,c,base)	SAVE_32FPRS(n, base)
#define __REST_32FPRS_VSRS(n,c,base)	REST_32FPRS(n, base)
#endif
#define SAVE_32FPRS_VSRS(n,c,base) \
	__SAVE_32FPRS_VSRS(n,__REG_##c,__REG_##base)
#define REST_32FPRS_VSRS(n,c,base) \
	__REST_32FPRS_VSRS(n,__REG_##c,__REG_##base)

/* Stack frame offsets for local variables. */
#define TM_FRAME_L0	TM_FRAME_SIZE-16
#define TM_FRAME_L1	TM_FRAME_SIZE-8


/* In order to access the TM SPRs, TM must be enabled.  So, do so: */
_GLOBAL(tm_enable)
	mfmsr	r4
	li	r3, MSR_TM >> 32
	sldi	r3, r3, 32
	and.	r0, r4, r3
	bne	1f
	or	r4, r4, r3
	mtmsrd	r4
1:	blr

_GLOBAL(tm_save_sprs)
	mfspr	r0, SPRN_TFHAR
	std	r0, THREAD_TM_TFHAR(r3)
	mfspr	r0, SPRN_TEXASR
	std	r0, THREAD_TM_TEXASR(r3)
	mfspr	r0, SPRN_TFIAR
	std	r0, THREAD_TM_TFIAR(r3)
	blr

_GLOBAL(tm_restore_sprs)
	ld	r0, THREAD_TM_TFHAR(r3)
	mtspr	SPRN_TFHAR, r0
	ld	r0, THREAD_TM_TEXASR(r3)
	mtspr	SPRN_TEXASR, r0
	ld	r0, THREAD_TM_TFIAR(r3)
	mtspr	SPRN_TFIAR, r0
	blr

	/* Passed an 8-bit failure cause as first argument. */
_GLOBAL(tm_abort)
	TABORT(R3)
	blr

/* void tm_reclaim(struct thread_struct *thread,
 *                 unsigned long orig_msr,
 *		   uint8_t cause)
 *
 *	- Performs a full reclaim.  This destroys outstanding
 *	  transactions and updates thread->regs.tm_ckpt_* with the
 *	  original checkpointed state.  Note that thread->regs is
 *	  unchanged.
 *	- FP regs are written back to thread->transact_fpr before
 *	  reclaiming.  These are the transactional (current) versions.
 *
 * Purpose is to both abort transactions of, and preserve the state of,
 * a transactions at a context switch. We preserve/restore both sets of process
 * state to restore them when the thread's scheduled again.  We continue in
 * userland as though nothing happened, but when the transaction is resumed
 * they will abort back to the checkpointed state we save out here.
 *
 * Call with IRQs off, stacks get all out of sync for some periods in here!
 */
_GLOBAL(tm_reclaim)
	mfcr	r6
	mflr	r0
	stw	r6, 8(r1)
	std	r0, 16(r1)
	std	r2, STK_GOT(r1)
	stdu	r1, -TM_FRAME_SIZE(r1)

	/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD]. */

	std	r3, STK_PARAM(R3)(r1)
	SAVE_NVGPRS(r1)

	/* We need to setup MSR for VSX register save instructions. */
	mfmsr	r14
	mr	r15, r14
	ori	r15, r15, MSR_FP
	li	r16, 0
	ori	r16, r16, MSR_EE /* IRQs hard off */
	andc	r15, r15, r16
	oris	r15, r15, MSR_VEC@h
#ifdef CONFIG_VSX
	BEGIN_FTR_SECTION
	oris	r15,r15, MSR_VSX@h
	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	mtmsrd	r15
	std	r14, TM_FRAME_L0(r1)

	/* Stash the stack pointer away for use after reclaim */
	std	r1, PACAR1(r13)

	/* ******************** FPR/VR/VSRs ************
	 * Before reclaiming, capture the current/transactional FPR/VR
	* versions /if used/.
	 *
	 * (If VSX used, FP and VMX are implied.  Or, we don't need to look
	 * at MSR.VSX as copying FP regs if .FP, vector regs if .VMX covers it.)
	 *
	 * We're passed the thread's MSR as parameter 2.
	 *
	 * We enabled VEC/FP/VSX in the msr above, so we can execute these
	 * instructions!
	 */
	andis.		r0, r4, MSR_VEC@h
	beq	dont_backup_vec

	addi	r7, r3, THREAD_TRANSACT_VRSTATE
	SAVE_32VRS(0, r6, r7)	/* r6 scratch, r7 transact vr state */
	mfvscr	v0
	li	r6, VRSTATE_VSCR
	stvx	v0, r7, r6
dont_backup_vec:
	mfspr	r0, SPRN_VRSAVE
	std	r0, THREAD_TRANSACT_VRSAVE(r3)

	andi.	r0, r4, MSR_FP
	beq	dont_backup_fp

	addi	r7, r3, THREAD_TRANSACT_FPSTATE
	SAVE_32FPRS_VSRS(0, R6, R7)	/* r6 scratch, r7 transact fp state */

	mffs    fr0
	stfd    fr0,FPSTATE_FPSCR(r7)

dont_backup_fp:
	/* Do sanity check on MSR to make sure we are suspended */
	li	r7, (MSR_TS_S)@higher
	srdi	r6, r14, 32
	and	r6, r6, r7
1:	tdeqi   r6, 0
	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0

	/* Clear MSR RI since we are about to change r1, EE is already off. */
	li	r4, 0
	mtmsrd	r4, 1

	/*
	 * BE CAREFUL HERE:
	 * At this point we can't take an SLB miss since we have MSR_RI
	 * off. Load only to/from the stack/paca which are in SLB bolted regions
	 * until we turn MSR RI back on.
	 *
	 * The moment we treclaim, ALL of our GPRs will switch
	 * to user register state.  (FPRs, CCR etc. also!)
	 * Use an sprg and a tm_scratch in the PACA to shuffle.
	 */
	TRECLAIM(R5)				/* Cause in r5 */

	/* ******************** GPRs ******************** */
	/* Stash the checkpointed r13 away in the scratch SPR and get the real
	 *  paca
	 */
	SET_SCRATCH0(r13)
	GET_PACA(r13)

	/* Stash the checkpointed r1 away in paca tm_scratch and get the real
	 * stack pointer back
	 */
	std	r1, PACATMSCRATCH(r13)
	ld	r1, PACAR1(r13)

	/* Store the PPR in r11 and reset to decent value */
	std	r11, GPR11(r1)			/* Temporary stash */

	/* Reset MSR RI so we can take SLB faults again */
	li	r11, MSR_RI
	mtmsrd	r11, 1

	mfspr	r11, SPRN_PPR
	HMT_MEDIUM

	/* Now get some more GPRS free */
	std	r7, GPR7(r1)			/* Temporary stash */
	std	r12, GPR12(r1)			/* ''   ''    ''   */
	ld	r12, STK_PARAM(R3)(r1)		/* Param 0, thread_struct * */

	std	r11, THREAD_TM_PPR(r12)		/* Store PPR and free r11 */

	addi	r7, r12, PT_CKPT_REGS		/* Thread's ckpt_regs */

	/* Make r7 look like an exception frame so that we
	 * can use the neat GPRx(n) macros.  r7 is NOT a pt_regs ptr!
	 */
	subi	r7, r7, STACK_FRAME_OVERHEAD

	/* Sync the userland GPRs 2-12, 14-31 to thread->regs: */
	SAVE_GPR(0, r7)				/* user r0 */
	SAVE_GPR(2, r7)			/* user r2 */
	SAVE_4GPRS(3, r7)			/* user r3-r6 */
	SAVE_GPR(8, r7)				/* user r8 */
	SAVE_GPR(9, r7)				/* user r9 */
	SAVE_GPR(10, r7)			/* user r10 */
	ld	r3, PACATMSCRATCH(r13)		/* user r1 */
	ld	r4, GPR7(r1)			/* user r7 */
	ld	r5, GPR11(r1)			/* user r11 */
	ld	r6, GPR12(r1)			/* user r12 */
	GET_SCRATCH0(8)				/* user r13 */
	std	r3, GPR1(r7)
	std	r4, GPR7(r7)
	std	r5, GPR11(r7)
	std	r6, GPR12(r7)
	std	r8, GPR13(r7)

	SAVE_NVGPRS(r7)				/* user r14-r31 */

	/* ******************** NIP ******************** */
	mfspr	r3, SPRN_TFHAR
	std	r3, _NIP(r7)			/* Returns to failhandler */
	/* The checkpointed NIP is ignored when rescheduling/rechkpting,
	 * but is used in signal return to 'wind back' to the abort handler.
	 */

	/* ******************** CR,LR,CCR,MSR ********** */
	mfctr	r3
	mflr	r4
	mfcr	r5
	mfxer	r6

	std	r3, _CTR(r7)
	std	r4, _LINK(r7)
	std	r5, _CCR(r7)
	std	r6, _XER(r7)


	/* ******************** TAR, DSCR ********** */
	mfspr	r3, SPRN_TAR
	mfspr	r4, SPRN_DSCR

	std	r3, THREAD_TM_TAR(r12)
	std	r4, THREAD_TM_DSCR(r12)

	/* MSR and flags:  We don't change CRs, and we don't need to alter
	 * MSR.
	 */

	/* TM regs, incl TEXASR -- these live in thread_struct.  Note they've
	 * been updated by the treclaim, to explain to userland the failure
	 * cause (aborted).
	 */
	mfspr	r0, SPRN_TEXASR
	mfspr	r3, SPRN_TFHAR
	mfspr	r4, SPRN_TFIAR
	std	r0, THREAD_TM_TEXASR(r12)
	std	r3, THREAD_TM_TFHAR(r12)
	std	r4, THREAD_TM_TFIAR(r12)

	/* AMR is checkpointed too, but is unsupported by Linux. */

	/* Restore original MSR/IRQ state & clear TM mode */
	ld	r14, TM_FRAME_L0(r1)		/* Orig MSR */
	li	r15, 0
	rldimi  r14, r15, MSR_TS_LG, (63-MSR_TS_LG)-1
	mtmsrd  r14

	REST_NVGPRS(r1)

	addi    r1, r1, TM_FRAME_SIZE
	lwz	r4, 8(r1)
	ld	r0, 16(r1)
	mtcr	r4
	mtlr	r0
	ld	r2, STK_GOT(r1)

	/* Load CPU's default DSCR */
	ld	r0, PACA_DSCR_DEFAULT(r13)
	mtspr	SPRN_DSCR, r0

	blr


	/* void tm_recheckpoint(struct thread_struct *thread,
	 *			unsigned long orig_msr)
	 *	- Restore the checkpointed register state saved by tm_reclaim
	 *	  when we switch_to a process.
	 *
	 *	Call with IRQs off, stacks get all out of sync for
	 *	some periods in here!
	 */
_GLOBAL(__tm_recheckpoint)
	mfcr	r5
	mflr	r0
	stw	r5, 8(r1)
	std	r0, 16(r1)
	std	r2, STK_GOT(r1)
	stdu	r1, -TM_FRAME_SIZE(r1)

	/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD].
	 * This is used for backing up the NVGPRs:
	 */
	SAVE_NVGPRS(r1)

	/* Load complete register state from ts_ckpt* registers */

	addi	r7, r3, PT_CKPT_REGS		/* Thread's ckpt_regs */

	/* Make r7 look like an exception frame so that we
	 * can use the neat GPRx(n) macros.  r7 is now NOT a pt_regs ptr!
	 */
	subi	r7, r7, STACK_FRAME_OVERHEAD

	SET_SCRATCH0(r1)

	mfmsr	r6
	/* R4 = original MSR to indicate whether thread used FP/Vector etc. */

	/* Enable FP/vec in MSR if necessary! */
	lis	r5, MSR_VEC@h
	ori	r5, r5, MSR_FP
	and.	r5, r4, r5
	beq	restore_gprs			/* if neither, skip both */

#ifdef CONFIG_VSX
	BEGIN_FTR_SECTION
	oris	r5, r5, MSR_VSX@h
	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	or	r5, r6, r5			/* Set MSR.FP+.VSX/.VEC */
	mtmsr	r5

#ifdef CONFIG_ALTIVEC
	/* FP and VEC registers:  These are recheckpointed from thread.fpr[]
	 * and thread.vr[] respectively.  The thread.transact_fpr[] version
	 * is more modern, and will be loaded subsequently by any FPUnavailable
	 * trap.
	 */
	andis.	r0, r4, MSR_VEC@h
	beq	dont_restore_vec

	addi	r8, r3, THREAD_VRSTATE
	li	r5, VRSTATE_VSCR
	lvx	v0, r8, r5
	mtvscr	v0
	REST_32VRS(0, r5, r8)			/* r5 scratch, r8 ptr */
dont_restore_vec:
	ld	r5, THREAD_VRSAVE(r3)
	mtspr	SPRN_VRSAVE, r5
#endif

	andi.	r0, r4, MSR_FP
	beq	dont_restore_fp

	addi	r8, r3, THREAD_FPSTATE
	lfd	fr0, FPSTATE_FPSCR(r8)
	MTFSF_L(fr0)
	REST_32FPRS_VSRS(0, R4, R8)

dont_restore_fp:
	mtmsr	r6				/* FP/Vec off again! */

restore_gprs:

	/* ******************** CR,LR,CCR,MSR ********** */
	ld	r4, _CTR(r7)
	ld	r5, _LINK(r7)
	ld	r8, _XER(r7)

	mtctr	r4
	mtlr	r5
	mtxer	r8

	/* ******************** TAR ******************** */
	ld	r4, THREAD_TM_TAR(r3)
	mtspr	SPRN_TAR,	r4

	/* Load up the PPR and DSCR in GPRs only at this stage */
	ld	r5, THREAD_TM_DSCR(r3)
	ld	r6, THREAD_TM_PPR(r3)

	REST_GPR(0, r7)				/* GPR0 */
	REST_2GPRS(2, r7)			/* GPR2-3 */
	REST_GPR(4, r7)				/* GPR4 */
	REST_4GPRS(8, r7)			/* GPR8-11 */
	REST_2GPRS(12, r7)			/* GPR12-13 */

	REST_NVGPRS(r7)				/* GPR14-31 */

	/* Load up PPR and DSCR here so we don't run with user values for long
	 */
	mtspr	SPRN_DSCR, r5
	mtspr	SPRN_PPR, r6

	/* Do final sanity check on TEXASR to make sure FS is set.  Do this
	 * here before we load up the userspace r1 so any bugs we hit will get
	 * a call chain */
	mfspr	r5, SPRN_TEXASR
	srdi	r5, r5, 16
	li	r6, (TEXASR_FS)@h
	and	r6, r6, r5
1:	tdeqi	r6, 0
	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0

	/* Do final sanity check on MSR to make sure we are not transactional
	 * or suspended
	 */
	mfmsr   r6
	li	r5, (MSR_TS_MASK)@higher
	srdi	r6, r6, 32
	and	r6, r6, r5
1:	tdnei   r6, 0
	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0

	/* Restore CR */
	ld	r6, _CCR(r7)
	mtcr    r6

	REST_GPR(6, r7)

	/*
	 * Store r1 and r5 on the stack so that we can access them
	 * after we clear MSR RI.
	 */

	REST_GPR(5, r7)
	std	r5, -8(r1)
	ld	r5, GPR1(r7)
	std	r5, -16(r1)

	REST_GPR(7, r7)

	/* Clear MSR RI since we are about to change r1. EE is already off */
	li	r5, 0
	mtmsrd	r5, 1

	/*
	 * BE CAREFUL HERE:
	 * At this point we can't take an SLB miss since we have MSR_RI
	 * off. Load only to/from the stack/paca which are in SLB bolted regions
	 * until we turn MSR RI back on.
	 */

	ld	r5, -8(r1)
	ld	r1, -16(r1)

	/* Commit register state as checkpointed state: */
	TRECHKPT

	HMT_MEDIUM

	/* Our transactional state has now changed.
	 *
	 * Now just get out of here.  Transactional (current) state will be
	 * updated once restore is called on the return path in the _switch-ed
	 * -to process.
	 */

	GET_PACA(r13)
	GET_SCRATCH0(r1)

	/* R1 is restored, so we are recoverable again.  EE is still off */
	li	r4, MSR_RI
	mtmsrd	r4, 1

	REST_NVGPRS(r1)

	addi    r1, r1, TM_FRAME_SIZE
	lwz	r4, 8(r1)
	ld	r0, 16(r1)
	mtcr	r4
	mtlr	r0
	ld	r2, STK_GOT(r1)

	/* Load CPU's default DSCR */
	ld	r0, PACA_DSCR_DEFAULT(r13)
	mtspr	SPRN_DSCR, r0

	blr

	/* ****************************************************************** */
Commit	Line	Data
98ae22e1 MN	1	/*
	2	* Transactional memory support routines to reclaim and recheckpoint
	3	* transactional process state.
	4	*
	5	* Copyright 2012 Matt Evans & Michael Neuling, IBM Corporation.
	6	*/
	7
	8	#include <asm/asm-offsets.h>
	9	#include <asm/ppc_asm.h>
	10	#include <asm/ppc-opcode.h>
	11	#include <asm/ptrace.h>
	12	#include <asm/reg.h>
7f06f21d	13	#include <asm/bug.h>
98ae22e1 MN	14
98ae22e1 MN	15	#ifdef CONFIG_VSX
de79f7b9 PM	16	/* See fpu.S, this is borrowed from there */
de79f7b9 PM	17	#define __SAVE_32FPRS_VSRS(n,c,base) \
98ae22e1 MN	18	BEGIN_FTR_SECTION \
	19	b 2f; \
	20	END_FTR_SECTION_IFSET(CPU_FTR_VSX); \
de79f7b9	21	SAVE_32FPRS(n,base); \
98ae22e1	22	b 3f; \
de79f7b9	23	2: SAVE_32VSRS(n,c,base); \
98ae22e1	24	3:
98ae22e1 MN	25	#define __REST_32FPRS_VSRS(n,c,base) \
	26	BEGIN_FTR_SECTION \
	27	b 2f; \
	28	END_FTR_SECTION_IFSET(CPU_FTR_VSX); \
	29	REST_32FPRS(n,base); \
	30	b 3f; \
	31	2: REST_32VSRS(n,c,base); \
	32	3:
	33	#else
de79f7b9 PM	34	#define __SAVE_32FPRS_VSRS(n,c,base) SAVE_32FPRS(n, base)
de79f7b9 PM	35	#define __REST_32FPRS_VSRS(n,c,base) REST_32FPRS(n, base)
98ae22e1	36	#endif
de79f7b9 PM	37	#define SAVE_32FPRS_VSRS(n,c,base) \
de79f7b9 PM	38	__SAVE_32FPRS_VSRS(n,__REG_##c,__REG_##base)
98ae22e1 MN	39	#define REST_32FPRS_VSRS(n,c,base) \
	40	__REST_32FPRS_VSRS(n,__REG_##c,__REG_##base)
	41
	42	/* Stack frame offsets for local variables. */
	43	#define TM_FRAME_L0 TM_FRAME_SIZE-16
	44	#define TM_FRAME_L1 TM_FRAME_SIZE-8
98ae22e1 MN	45
	46
	47	/* In order to access the TM SPRs, TM must be enabled. So, do so: */
	48	_GLOBAL(tm_enable)
	49	mfmsr r4
	50	li r3, MSR_TM >> 32
	51	sldi r3, r3, 32
	52	and. r0, r4, r3
	53	bne 1f
	54	or r4, r4, r3
	55	mtmsrd r4
	56	1: blr
	57
	58	_GLOBAL(tm_save_sprs)
	59	mfspr r0, SPRN_TFHAR
	60	std r0, THREAD_TM_TFHAR(r3)
	61	mfspr r0, SPRN_TEXASR
	62	std r0, THREAD_TM_TEXASR(r3)
	63	mfspr r0, SPRN_TFIAR
	64	std r0, THREAD_TM_TFIAR(r3)
	65	blr
	66
	67	_GLOBAL(tm_restore_sprs)
	68	ld r0, THREAD_TM_TFHAR(r3)
	69	mtspr SPRN_TFHAR, r0
	70	ld r0, THREAD_TM_TEXASR(r3)
	71	mtspr SPRN_TEXASR, r0
	72	ld r0, THREAD_TM_TFIAR(r3)
	73	mtspr SPRN_TFIAR, r0
	74	blr
	75
	76	/* Passed an 8-bit failure cause as first argument. */
	77	_GLOBAL(tm_abort)
	78	TABORT(R3)
	79	blr
	80
98ae22e1 MN	81	/* void tm_reclaim(struct thread_struct *thread,
	82	* unsigned long orig_msr,
	83	* uint8_t cause)
	84	*
	85	* - Performs a full reclaim. This destroys outstanding
	86	* transactions and updates thread->regs.tm_ckpt_* with the
	87	* original checkpointed state. Note that thread->regs is
	88	* unchanged.
	89	* - FP regs are written back to thread->transact_fpr before
	90	* reclaiming. These are the transactional (current) versions.
	91	*
	92	* Purpose is to both abort transactions of, and preserve the state of,
	93	* a transactions at a context switch. We preserve/restore both sets of process
	94	* state to restore them when the thread's scheduled again. We continue in
	95	* userland as though nothing happened, but when the transaction is resumed
	96	* they will abort back to the checkpointed state we save out here.
	97	*
	98	* Call with IRQs off, stacks get all out of sync for some periods in here!
	99	*/
	100	_GLOBAL(tm_reclaim)
	101	mfcr r6
	102	mflr r0
bbe30b3b	103	stw r6, 8(r1)
98ae22e1	104	std r0, 16(r1)
6403105b	105	std r2, STK_GOT(r1)
98ae22e1 MN	106	stdu r1, -TM_FRAME_SIZE(r1)
	107
	108	/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD]. */
	109
c2e31bdc	110	std r3, STK_PARAM(R3)(r1)
98ae22e1 MN	111	SAVE_NVGPRS(r1)
98ae22e1 MN	112
190ce869	113	/* We need to setup MSR for VSX register save instructions. */
98ae22e1 MN	114	mfmsr r14
	115	mr r15, r14
	116	ori r15, r15, MSR_FP
190ce869	117	li r16, 0
c69e63b0	118	ori r16, r16, MSR_EE /* IRQs hard off */
090b9284	119	andc r15, r15, r16
98ae22e1 MN	120	oris r15, r15, MSR_VEC@h
	121	#ifdef CONFIG_VSX
	122	BEGIN_FTR_SECTION
	123	oris r15,r15, MSR_VSX@h
	124	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
	125	#endif
	126	mtmsrd r15
	127	std r14, TM_FRAME_L0(r1)
	128
	129	/* Stash the stack pointer away for use after reclaim */
	130	std r1, PACAR1(r13)
	131
	132	/* ****************** FPR/VR/VSRs **********
	133	* Before reclaiming, capture the current/transactional FPR/VR
	134	* versions /if used/.
	135	*
	136	* (If VSX used, FP and VMX are implied. Or, we don't need to look
	137	* at MSR.VSX as copying FP regs if .FP, vector regs if .VMX covers it.)
	138	*
	139	* We're passed the thread's MSR as parameter 2.
	140	*
	141	* We enabled VEC/FP/VSX in the msr above, so we can execute these
	142	* instructions!
	143	*/
	144	andis. r0, r4, MSR_VEC@h
	145	beq dont_backup_vec
	146
de79f7b9 PM	147	addi r7, r3, THREAD_TRANSACT_VRSTATE
de79f7b9 PM	148	SAVE_32VRS(0, r6, r7) /* r6 scratch, r7 transact vr state */
c2ce6f9f	149	mfvscr v0
de79f7b9	150	li r6, VRSTATE_VSCR
c2ce6f9f	151	stvx v0, r7, r6
408a7e08	152	dont_backup_vec:
98ae22e1 MN	153	mfspr r0, SPRN_VRSAVE
	154	std r0, THREAD_TRANSACT_VRSAVE(r3)
	155
98ae22e1 MN	156	andi. r0, r4, MSR_FP
	157	beq dont_backup_fp
	158
de79f7b9 PM	159	addi r7, r3, THREAD_TRANSACT_FPSTATE
de79f7b9 PM	160	SAVE_32FPRS_VSRS(0, R6, R7) /* r6 scratch, r7 transact fp state */
98ae22e1 MN	161
98ae22e1 MN	162	mffs fr0
de79f7b9	163	stfd fr0,FPSTATE_FPSCR(r7)
98ae22e1 MN	164
98ae22e1 MN	165	dont_backup_fp:
7f06f21d MN	166	/* Do sanity check on MSR to make sure we are suspended */
	167	li r7, (MSR_TS_S)@higher
	168	srdi r6, r14, 32
	169	and r6, r6, r7
	170	1: tdeqi r6, 0
	171	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
	172
190ce869 MN	173	/* Clear MSR RI since we are about to change r1, EE is already off. */
	174	li r4, 0
	175	mtmsrd r4, 1
	176
	177	/*
	178	* BE CAREFUL HERE:
	179	* At this point we can't take an SLB miss since we have MSR_RI
	180	* off. Load only to/from the stack/paca which are in SLB bolted regions
	181	* until we turn MSR RI back on.
	182	*
	183	* The moment we treclaim, ALL of our GPRs will switch
98ae22e1 MN	184	* to user register state. (FPRs, CCR etc. also!)
	185	* Use an sprg and a tm_scratch in the PACA to shuffle.
	186	*/
	187	TRECLAIM(R5) /* Cause in r5 */
	188
	189	/* ****************** GPRs ****************** */
	190	/* Stash the checkpointed r13 away in the scratch SPR and get the real
	191	* paca
	192	*/
	193	SET_SCRATCH0(r13)
	194	GET_PACA(r13)
	195
	196	/* Stash the checkpointed r1 away in paca tm_scratch and get the real
	197	* stack pointer back
	198	*/
	199	std r1, PACATMSCRATCH(r13)
	200	ld r1, PACAR1(r13)
	201
e9bdc3d6 MN	202	/* Store the PPR in r11 and reset to decent value */
e9bdc3d6 MN	203	std r11, GPR11(r1) /* Temporary stash */
190ce869 MN	204
	205	/* Reset MSR RI so we can take SLB faults again */
	206	li r11, MSR_RI
	207	mtmsrd r11, 1
	208
e9bdc3d6 MN	209	mfspr r11, SPRN_PPR
	210	HMT_MEDIUM
	211
98ae22e1 MN	212	/* Now get some more GPRS free */
	213	std r7, GPR7(r1) /* Temporary stash */
	214	std r12, GPR12(r1) /* '' '' '' */
c2e31bdc	215	ld r12, STK_PARAM(R3)(r1) /* Param 0, thread_struct * */
98ae22e1	216
e9bdc3d6 MN	217	std r11, THREAD_TM_PPR(r12) /* Store PPR and free r11 */
e9bdc3d6 MN	218
98ae22e1 MN	219	addi r7, r12, PT_CKPT_REGS /* Thread's ckpt_regs */
	220
	221	/* Make r7 look like an exception frame so that we
	222	* can use the neat GPRx(n) macros. r7 is NOT a pt_regs ptr!
	223	*/
	224	subi r7, r7, STACK_FRAME_OVERHEAD
	225
	226	/* Sync the userland GPRs 2-12, 14-31 to thread->regs: */
	227	SAVE_GPR(0, r7) /* user r0 */
	228	SAVE_GPR(2, r7) /* user r2 */
	229	SAVE_4GPRS(3, r7) /* user r3-r6 */
e9bdc3d6 MN	230	SAVE_GPR(8, r7) /* user r8 */
	231	SAVE_GPR(9, r7) /* user r9 */
	232	SAVE_GPR(10, r7) /* user r10 */
98ae22e1 MN	233	ld r3, PACATMSCRATCH(r13) /* user r1 */
98ae22e1 MN	234	ld r4, GPR7(r1) /* user r7 */
e9bdc3d6 MN	235	ld r5, GPR11(r1) /* user r11 */
	236	ld r6, GPR12(r1) /* user r12 */
	237	GET_SCRATCH0(8) /* user r13 */
98ae22e1 MN	238	std r3, GPR1(r7)
98ae22e1 MN	239	std r4, GPR7(r7)
e9bdc3d6 MN	240	std r5, GPR11(r7)
	241	std r6, GPR12(r7)
	242	std r8, GPR13(r7)
98ae22e1 MN	243
	244	SAVE_NVGPRS(r7) /* user r14-r31 */
	245
	246	/* ****************** NIP ****************** */
	247	mfspr r3, SPRN_TFHAR
	248	std r3, _NIP(r7) /* Returns to failhandler */
	249	/* The checkpointed NIP is ignored when rescheduling/rechkpting,
	250	* but is used in signal return to 'wind back' to the abort handler.
	251	*/
	252
	253	/* ****************** CR,LR,CCR,MSR ******** */
	254	mfctr r3
	255	mflr r4
	256	mfcr r5
	257	mfxer r6
	258
	259	std r3, _CTR(r7)
	260	std r4, _LINK(r7)
	261	std r5, _CCR(r7)
	262	std r6, _XER(r7)
	263
28e61cc4	264
e9bdc3d6	265	/* ****************** TAR, DSCR ******** */
28e61cc4	266	mfspr r3, SPRN_TAR
e9bdc3d6	267	mfspr r4, SPRN_DSCR
28e61cc4 MN	268
28e61cc4 MN	269	std r3, THREAD_TM_TAR(r12)
e9bdc3d6	270	std r4, THREAD_TM_DSCR(r12)
28e61cc4	271
98ae22e1 MN	272	/* MSR and flags: We don't change CRs, and we don't need to alter
	273	* MSR.
	274	*/
	275
	276	/* TM regs, incl TEXASR -- these live in thread_struct. Note they've
	277	* been updated by the treclaim, to explain to userland the failure
	278	* cause (aborted).
	279	*/
	280	mfspr r0, SPRN_TEXASR
	281	mfspr r3, SPRN_TFHAR
	282	mfspr r4, SPRN_TFIAR
	283	std r0, THREAD_TM_TEXASR(r12)
	284	std r3, THREAD_TM_TFHAR(r12)
	285	std r4, THREAD_TM_TFIAR(r12)
	286
e9bdc3d6	287	/* AMR is checkpointed too, but is unsupported by Linux. */
98ae22e1 MN	288
	289	/* Restore original MSR/IRQ state & clear TM mode */
	290	ld r14, TM_FRAME_L0(r1) /* Orig MSR */
	291	li r15, 0
	292	rldimi r14, r15, MSR_TS_LG, (63-MSR_TS_LG)-1
	293	mtmsrd r14
	294
	295	REST_NVGPRS(r1)
	296
	297	addi r1, r1, TM_FRAME_SIZE
bbe30b3b	298	lwz r4, 8(r1)
98ae22e1 MN	299	ld r0, 16(r1)
	300	mtcr r4
	301	mtlr r0
6403105b	302	ld r2, STK_GOT(r1)
e9bdc3d6	303
1739ea9e	304	/* Load CPU's default DSCR */
1db36525	305	ld r0, PACA_DSCR_DEFAULT(r13)
e9bdc3d6 MN	306	mtspr SPRN_DSCR, r0
e9bdc3d6 MN	307
98ae22e1 MN	308	blr
	309
	310
	311	/* void tm_recheckpoint(struct thread_struct *thread,
	312	* unsigned long orig_msr)
	313	* - Restore the checkpointed register state saved by tm_reclaim
	314	* when we switch_to a process.
	315	*
	316	* Call with IRQs off, stacks get all out of sync for
	317	* some periods in here!
	318	*/
e6b8fd02	319	_GLOBAL(__tm_recheckpoint)
98ae22e1 MN	320	mfcr r5
98ae22e1 MN	321	mflr r0
bbe30b3b	322	stw r5, 8(r1)
98ae22e1	323	std r0, 16(r1)
6403105b	324	std r2, STK_GOT(r1)
98ae22e1 MN	325	stdu r1, -TM_FRAME_SIZE(r1)
	326
	327	/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD].
	328	* This is used for backing up the NVGPRs:
	329	*/
	330	SAVE_NVGPRS(r1)
	331
98ae22e1 MN	332	/* Load complete register state from ts_ckpt* registers */
	333
	334	addi r7, r3, PT_CKPT_REGS /* Thread's ckpt_regs */
	335
	336	/* Make r7 look like an exception frame so that we
	337	* can use the neat GPRx(n) macros. r7 is now NOT a pt_regs ptr!
	338	*/
	339	subi r7, r7, STACK_FRAME_OVERHEAD
	340
	341	SET_SCRATCH0(r1)
	342
	343	mfmsr r6
	344	/* R4 = original MSR to indicate whether thread used FP/Vector etc. */
	345
	346	/* Enable FP/vec in MSR if necessary! */
	347	lis r5, MSR_VEC@h
	348	ori r5, r5, MSR_FP
	349	and. r5, r4, r5
	350	beq restore_gprs /* if neither, skip both */
	351
	352	#ifdef CONFIG_VSX
	353	BEGIN_FTR_SECTION
	354	oris r5, r5, MSR_VSX@h
	355	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
	356	#endif
	357	or r5, r6, r5 /* Set MSR.FP+.VSX/.VEC */
	358	mtmsr r5
	359
f110c0c1	360	#ifdef CONFIG_ALTIVEC
98ae22e1 MN	361	/* FP and VEC registers: These are recheckpointed from thread.fpr[]
	362	* and thread.vr[] respectively. The thread.transact_fpr[] version
	363	* is more modern, and will be loaded subsequently by any FPUnavailable
	364	* trap.
	365	*/
	366	andis. r0, r4, MSR_VEC@h
	367	beq dont_restore_vec
	368
de79f7b9 PM	369	addi r8, r3, THREAD_VRSTATE
de79f7b9 PM	370	li r5, VRSTATE_VSCR
c2ce6f9f AB	371	lvx v0, r8, r5
c2ce6f9f AB	372	mtvscr v0
de79f7b9	373	REST_32VRS(0, r5, r8) /* r5 scratch, r8 ptr */
408a7e08	374	dont_restore_vec:
98ae22e1 MN	375	ld r5, THREAD_VRSAVE(r3)
98ae22e1 MN	376	mtspr SPRN_VRSAVE, r5
f110c0c1	377	#endif
98ae22e1	378
98ae22e1 MN	379	andi. r0, r4, MSR_FP
	380	beq dont_restore_fp
	381
de79f7b9 PM	382	addi r8, r3, THREAD_FPSTATE
de79f7b9 PM	383	lfd fr0, FPSTATE_FPSCR(r8)
98ae22e1	384	MTFSF_L(fr0)
de79f7b9	385	REST_32FPRS_VSRS(0, R4, R8)
98ae22e1 MN	386
	387	dont_restore_fp:
	388	mtmsr r6 /* FP/Vec off again! */
	389
	390	restore_gprs:
28e61cc4	391
e9bdc3d6 MN	392	/* ****************** CR,LR,CCR,MSR ******** */
	393	ld r4, _CTR(r7)
	394	ld r5, _LINK(r7)
e9bdc3d6	395	ld r8, _XER(r7)
28e61cc4	396
e9bdc3d6 MN	397	mtctr r4
e9bdc3d6 MN	398	mtlr r5
e9bdc3d6	399	mtxer r8
28e61cc4	400
e9bdc3d6 MN	401	/* ****************** TAR ****************** */
	402	ld r4, THREAD_TM_TAR(r3)
	403	mtspr SPRN_TAR, r4
98ae22e1	404
e9bdc3d6 MN	405	/* Load up the PPR and DSCR in GPRs only at this stage */
	406	ld r5, THREAD_TM_DSCR(r3)
	407	ld r6, THREAD_TM_PPR(r3)
98ae22e1	408
7f06f21d MN	409	REST_GPR(0, r7) /* GPR0 */
7f06f21d MN	410	REST_2GPRS(2, r7) /* GPR2-3 */
e9bdc3d6	411	REST_GPR(4, r7) /* GPR4 */
98ae22e1 MN	412	REST_4GPRS(8, r7) /* GPR8-11 */
	413	REST_2GPRS(12, r7) /* GPR12-13 */
	414
	415	REST_NVGPRS(r7) /* GPR14-31 */
	416
e9bdc3d6 MN	417	/* Load up PPR and DSCR here so we don't run with user values for long
	418	*/
	419	mtspr SPRN_DSCR, r5
	420	mtspr SPRN_PPR, r6
	421
7f06f21d MN	422	/* Do final sanity check on TEXASR to make sure FS is set. Do this
	423	* here before we load up the userspace r1 so any bugs we hit will get
	424	* a call chain */
	425	mfspr r5, SPRN_TEXASR
	426	srdi r5, r5, 16
	427	li r6, (TEXASR_FS)@h
	428	and r6, r6, r5
	429	1: tdeqi r6, 0
	430	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
	431
	432	/* Do final sanity check on MSR to make sure we are not transactional
	433	* or suspended
	434	*/
	435	mfmsr r6
	436	li r5, (MSR_TS_MASK)@higher
	437	srdi r6, r6, 32
	438	and r6, r6, r5
	439	1: tdnei r6, 0
	440	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
	441
	442	/* Restore CR */
	443	ld r6, _CCR(r7)
	444	mtcr r6
	445
e9bdc3d6	446	REST_GPR(6, r7)
190ce869 MN	447
	448	/*
	449	* Store r1 and r5 on the stack so that we can access them
	450	* after we clear MSR RI.
	451	*/
	452
	453	REST_GPR(5, r7)
	454	std r5, -8(r1)
	455	ld r5, GPR1(r7)
	456	std r5, -16(r1)
	457
	458	REST_GPR(7, r7)
	459
	460	/* Clear MSR RI since we are about to change r1. EE is already off */
	461	li r5, 0
	462	mtmsrd r5, 1
	463
	464	/*
	465	* BE CAREFUL HERE:
	466	* At this point we can't take an SLB miss since we have MSR_RI
	467	* off. Load only to/from the stack/paca which are in SLB bolted regions
	468	* until we turn MSR RI back on.
	469	*/
	470
	471	ld r5, -8(r1)
	472	ld r1, -16(r1)
98ae22e1 MN	473
	474	/* Commit register state as checkpointed state: */
	475	TRECHKPT
	476
e9bdc3d6 MN	477	HMT_MEDIUM
e9bdc3d6 MN	478
98ae22e1 MN	479	/* Our transactional state has now changed.
	480	*
	481	* Now just get out of here. Transactional (current) state will be
	482	* updated once restore is called on the return path in the _switch-ed
	483	* -to process.
	484	*/
	485
	486	GET_PACA(r13)
	487	GET_SCRATCH0(r1)
	488
090b9284 MN	489	/* R1 is restored, so we are recoverable again. EE is still off */
	490	li r4, MSR_RI
	491	mtmsrd r4, 1
	492
98ae22e1 MN	493	REST_NVGPRS(r1)
	494
	495	addi r1, r1, TM_FRAME_SIZE
bbe30b3b	496	lwz r4, 8(r1)
98ae22e1 MN	497	ld r0, 16(r1)
	498	mtcr r4
	499	mtlr r0
6403105b	500	ld r2, STK_GOT(r1)
e9bdc3d6	501
1739ea9e	502	/* Load CPU's default DSCR */
1db36525	503	ld r0, PACA_DSCR_DEFAULT(r13)
e9bdc3d6 MN	504	mtspr SPRN_DSCR, r0
e9bdc3d6 MN	505
98ae22e1 MN	506	blr
	507
	508	/* ****************************************************************** */