[linux-2.6-block.git] / arch / powerpc / kernel / tm.S

/*
 * Transactional memory support routines to reclaim and recheckpoint
 * transactional process state.
 *
 * Copyright 2012 Matt Evans & Michael Neuling, IBM Corporation.
 */

#include <asm/asm-offsets.h>
#include <asm/ppc_asm.h>
#include <asm/ppc-opcode.h>
#include <asm/ptrace.h>
#include <asm/reg.h>
#include <asm/bug.h>

#ifdef CONFIG_VSX
/* See fpu.S, this is borrowed from there */
#define __SAVE_32FPRS_VSRS(n,c,base)		\
BEGIN_FTR_SECTION				\
	b	2f;				\
END_FTR_SECTION_IFSET(CPU_FTR_VSX);		\
	SAVE_32FPRS(n,base);			\
	b	3f;				\
2:	SAVE_32VSRS(n,c,base);			\
3:
#define __REST_32FPRS_VSRS(n,c,base)		\
BEGIN_FTR_SECTION				\
	b	2f;				\
END_FTR_SECTION_IFSET(CPU_FTR_VSX);		\
	REST_32FPRS(n,base);			\
	b	3f;				\
2:	REST_32VSRS(n,c,base);			\
3:
#else
#define __SAVE_32FPRS_VSRS(n,c,base)	SAVE_32FPRS(n, base)
#define __REST_32FPRS_VSRS(n,c,base)	REST_32FPRS(n, base)
#endif
#define SAVE_32FPRS_VSRS(n,c,base) \
	__SAVE_32FPRS_VSRS(n,__REG_##c,__REG_##base)
#define REST_32FPRS_VSRS(n,c,base) \
	__REST_32FPRS_VSRS(n,__REG_##c,__REG_##base)

/* Stack frame offsets for local variables. */
#define TM_FRAME_L0	TM_FRAME_SIZE-16
#define TM_FRAME_L1	TM_FRAME_SIZE-8


/* In order to access the TM SPRs, TM must be enabled.  So, do so: */
_GLOBAL(tm_enable)
	mfmsr	r4
	li	r3, MSR_TM >> 32
	sldi	r3, r3, 32
	and.	r0, r4, r3
	bne	1f
	or	r4, r4, r3
	mtmsrd	r4
1:	blr

_GLOBAL(tm_save_sprs)
	mfspr	r0, SPRN_TFHAR
	std	r0, THREAD_TM_TFHAR(r3)
	mfspr	r0, SPRN_TEXASR
	std	r0, THREAD_TM_TEXASR(r3)
	mfspr	r0, SPRN_TFIAR
	std	r0, THREAD_TM_TFIAR(r3)
	blr

_GLOBAL(tm_restore_sprs)
	ld	r0, THREAD_TM_TFHAR(r3)
	mtspr	SPRN_TFHAR, r0
	ld	r0, THREAD_TM_TEXASR(r3)
	mtspr	SPRN_TEXASR, r0
	ld	r0, THREAD_TM_TFIAR(r3)
	mtspr	SPRN_TFIAR, r0
	blr

	/* Passed an 8-bit failure cause as first argument. */
_GLOBAL(tm_abort)
	TABORT(R3)
	blr

/* void tm_reclaim(struct thread_struct *thread,
 *                 unsigned long orig_msr,
 *		   uint8_t cause)
 *
 *	- Performs a full reclaim.  This destroys outstanding
 *	  transactions and updates thread->regs.tm_ckpt_* with the
 *	  original checkpointed state.  Note that thread->regs is
 *	  unchanged.
 *	- FP regs are written back to thread->transact_fpr before
 *	  reclaiming.  These are the transactional (current) versions.
 *
 * Purpose is to both abort transactions of, and preserve the state of,
 * a transactions at a context switch. We preserve/restore both sets of process
 * state to restore them when the thread's scheduled again.  We continue in
 * userland as though nothing happened, but when the transaction is resumed
 * they will abort back to the checkpointed state we save out here.
 *
 * Call with IRQs off, stacks get all out of sync for some periods in here!
 */
_GLOBAL(tm_reclaim)
	mfcr	r6
	mflr	r0
	stw	r6, 8(r1)
	std	r0, 16(r1)
	std	r2, STK_GOT(r1)
	stdu	r1, -TM_FRAME_SIZE(r1)

	/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD]. */

	std	r3, STK_PARAM(R3)(r1)
	SAVE_NVGPRS(r1)

	/* We need to setup MSR for VSX register save instructions.  Here we
	 * also clear the MSR RI since when we do the treclaim, we won't have a
	 * valid kernel pointer for a while.  We clear RI here as it avoids
	 * adding another mtmsr closer to the treclaim.  This makes the region
	 * maked as non-recoverable wider than it needs to be but it saves on
	 * inserting another mtmsrd later.
	 */
	mfmsr	r14
	mr	r15, r14
	ori	r15, r15, MSR_FP
	li	r16, MSR_RI
	ori	r16, r16, MSR_EE /* IRQs hard off */
	andc	r15, r15, r16
	oris	r15, r15, MSR_VEC@h
#ifdef CONFIG_VSX
	BEGIN_FTR_SECTION
	oris	r15,r15, MSR_VSX@h
	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	mtmsrd	r15
	std	r14, TM_FRAME_L0(r1)

	/* Stash the stack pointer away for use after reclaim */
	std	r1, PACAR1(r13)

	/* ******************** FPR/VR/VSRs ************
	 * Before reclaiming, capture the current/transactional FPR/VR
	* versions /if used/.
	 *
	 * (If VSX used, FP and VMX are implied.  Or, we don't need to look
	 * at MSR.VSX as copying FP regs if .FP, vector regs if .VMX covers it.)
	 *
	 * We're passed the thread's MSR as parameter 2.
	 *
	 * We enabled VEC/FP/VSX in the msr above, so we can execute these
	 * instructions!
	 */
	andis.		r0, r4, MSR_VEC@h
	beq	dont_backup_vec

	addi	r7, r3, THREAD_TRANSACT_VRSTATE
	SAVE_32VRS(0, r6, r7)	/* r6 scratch, r7 transact vr state */
	mfvscr	v0
	li	r6, VRSTATE_VSCR
	stvx	v0, r7, r6
dont_backup_vec:
	mfspr	r0, SPRN_VRSAVE
	std	r0, THREAD_TRANSACT_VRSAVE(r3)

	andi.	r0, r4, MSR_FP
	beq	dont_backup_fp

	addi	r7, r3, THREAD_TRANSACT_FPSTATE
	SAVE_32FPRS_VSRS(0, R6, R7)	/* r6 scratch, r7 transact fp state */

	mffs    fr0
	stfd    fr0,FPSTATE_FPSCR(r7)

dont_backup_fp:
	/* Do sanity check on MSR to make sure we are suspended */
	li	r7, (MSR_TS_S)@higher
	srdi	r6, r14, 32
	and	r6, r6, r7
1:	tdeqi   r6, 0
	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0

	/* The moment we treclaim, ALL of our GPRs will switch
	 * to user register state.  (FPRs, CCR etc. also!)
	 * Use an sprg and a tm_scratch in the PACA to shuffle.
	 */
	TRECLAIM(R5)				/* Cause in r5 */

	/* ******************** GPRs ******************** */
	/* Stash the checkpointed r13 away in the scratch SPR and get the real
	 *  paca
	 */
	SET_SCRATCH0(r13)
	GET_PACA(r13)

	/* Stash the checkpointed r1 away in paca tm_scratch and get the real
	 * stack pointer back
	 */
	std	r1, PACATMSCRATCH(r13)
	ld	r1, PACAR1(r13)

	/* Store the PPR in r11 and reset to decent value */
	std	r11, GPR11(r1)			/* Temporary stash */
	mfspr	r11, SPRN_PPR
	HMT_MEDIUM

	/* Now get some more GPRS free */
	std	r7, GPR7(r1)			/* Temporary stash */
	std	r12, GPR12(r1)			/* ''   ''    ''   */
	ld	r12, STK_PARAM(R3)(r1)		/* Param 0, thread_struct * */

	std	r11, THREAD_TM_PPR(r12)		/* Store PPR and free r11 */

	addi	r7, r12, PT_CKPT_REGS		/* Thread's ckpt_regs */

	/* Make r7 look like an exception frame so that we
	 * can use the neat GPRx(n) macros.  r7 is NOT a pt_regs ptr!
	 */
	subi	r7, r7, STACK_FRAME_OVERHEAD

	/* Sync the userland GPRs 2-12, 14-31 to thread->regs: */
	SAVE_GPR(0, r7)				/* user r0 */
	SAVE_GPR(2, r7)			/* user r2 */
	SAVE_4GPRS(3, r7)			/* user r3-r6 */
	SAVE_GPR(8, r7)				/* user r8 */
	SAVE_GPR(9, r7)				/* user r9 */
	SAVE_GPR(10, r7)			/* user r10 */
	ld	r3, PACATMSCRATCH(r13)		/* user r1 */
	ld	r4, GPR7(r1)			/* user r7 */
	ld	r5, GPR11(r1)			/* user r11 */
	ld	r6, GPR12(r1)			/* user r12 */
	GET_SCRATCH0(8)				/* user r13 */
	std	r3, GPR1(r7)
	std	r4, GPR7(r7)
	std	r5, GPR11(r7)
	std	r6, GPR12(r7)
	std	r8, GPR13(r7)

	SAVE_NVGPRS(r7)				/* user r14-r31 */

	/* ******************** NIP ******************** */
	mfspr	r3, SPRN_TFHAR
	std	r3, _NIP(r7)			/* Returns to failhandler */
	/* The checkpointed NIP is ignored when rescheduling/rechkpting,
	 * but is used in signal return to 'wind back' to the abort handler.
	 */

	/* ******************** CR,LR,CCR,MSR ********** */
	mfctr	r3
	mflr	r4
	mfcr	r5
	mfxer	r6

	std	r3, _CTR(r7)
	std	r4, _LINK(r7)
	std	r5, _CCR(r7)
	std	r6, _XER(r7)


	/* ******************** TAR, DSCR ********** */
	mfspr	r3, SPRN_TAR
	mfspr	r4, SPRN_DSCR

	std	r3, THREAD_TM_TAR(r12)
	std	r4, THREAD_TM_DSCR(r12)

	/* MSR and flags:  We don't change CRs, and we don't need to alter
	 * MSR.
	 */

	/* TM regs, incl TEXASR -- these live in thread_struct.  Note they've
	 * been updated by the treclaim, to explain to userland the failure
	 * cause (aborted).
	 */
	mfspr	r0, SPRN_TEXASR
	mfspr	r3, SPRN_TFHAR
	mfspr	r4, SPRN_TFIAR
	std	r0, THREAD_TM_TEXASR(r12)
	std	r3, THREAD_TM_TFHAR(r12)
	std	r4, THREAD_TM_TFIAR(r12)

	/* AMR is checkpointed too, but is unsupported by Linux. */

	/* Restore original MSR/IRQ state & clear TM mode */
	ld	r14, TM_FRAME_L0(r1)		/* Orig MSR */
	li	r15, 0
	rldimi  r14, r15, MSR_TS_LG, (63-MSR_TS_LG)-1
	mtmsrd  r14

	REST_NVGPRS(r1)

	addi    r1, r1, TM_FRAME_SIZE
	lwz	r4, 8(r1)
	ld	r0, 16(r1)
	mtcr	r4
	mtlr	r0
	ld	r2, STK_GOT(r1)

	/* Load CPU's default DSCR */
	ld	r0, PACA_DSCR_DEFAULT(r13)
	mtspr	SPRN_DSCR, r0

	blr


	/* void tm_recheckpoint(struct thread_struct *thread,
	 *			unsigned long orig_msr)
	 *	- Restore the checkpointed register state saved by tm_reclaim
	 *	  when we switch_to a process.
	 *
	 *	Call with IRQs off, stacks get all out of sync for
	 *	some periods in here!
	 */
_GLOBAL(__tm_recheckpoint)
	mfcr	r5
	mflr	r0
	stw	r5, 8(r1)
	std	r0, 16(r1)
	std	r2, STK_GOT(r1)
	stdu	r1, -TM_FRAME_SIZE(r1)

	/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD].
	 * This is used for backing up the NVGPRs:
	 */
	SAVE_NVGPRS(r1)

	/* Load complete register state from ts_ckpt* registers */

	addi	r7, r3, PT_CKPT_REGS		/* Thread's ckpt_regs */

	/* Make r7 look like an exception frame so that we
	 * can use the neat GPRx(n) macros.  r7 is now NOT a pt_regs ptr!
	 */
	subi	r7, r7, STACK_FRAME_OVERHEAD

	SET_SCRATCH0(r1)

	mfmsr	r6
	/* R4 = original MSR to indicate whether thread used FP/Vector etc. */

	/* Enable FP/vec in MSR if necessary! */
	lis	r5, MSR_VEC@h
	ori	r5, r5, MSR_FP
	and.	r5, r4, r5
	beq	restore_gprs			/* if neither, skip both */

#ifdef CONFIG_VSX
	BEGIN_FTR_SECTION
	oris	r5, r5, MSR_VSX@h
	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	or	r5, r6, r5			/* Set MSR.FP+.VSX/.VEC */
	mtmsr	r5

#ifdef CONFIG_ALTIVEC
	/* FP and VEC registers:  These are recheckpointed from thread.fpr[]
	 * and thread.vr[] respectively.  The thread.transact_fpr[] version
	 * is more modern, and will be loaded subsequently by any FPUnavailable
	 * trap.
	 */
	andis.	r0, r4, MSR_VEC@h
	beq	dont_restore_vec

	addi	r8, r3, THREAD_VRSTATE
	li	r5, VRSTATE_VSCR
	lvx	v0, r8, r5
	mtvscr	v0
	REST_32VRS(0, r5, r8)			/* r5 scratch, r8 ptr */
dont_restore_vec:
	ld	r5, THREAD_VRSAVE(r3)
	mtspr	SPRN_VRSAVE, r5
#endif

	andi.	r0, r4, MSR_FP
	beq	dont_restore_fp

	addi	r8, r3, THREAD_FPSTATE
	lfd	fr0, FPSTATE_FPSCR(r8)
	MTFSF_L(fr0)
	REST_32FPRS_VSRS(0, R4, R8)

dont_restore_fp:
	mtmsr	r6				/* FP/Vec off again! */

restore_gprs:

	/* ******************** CR,LR,CCR,MSR ********** */
	ld	r4, _CTR(r7)
	ld	r5, _LINK(r7)
	ld	r8, _XER(r7)

	mtctr	r4
	mtlr	r5
	mtxer	r8

	/* ******************** TAR ******************** */
	ld	r4, THREAD_TM_TAR(r3)
	mtspr	SPRN_TAR,	r4

	/* Load up the PPR and DSCR in GPRs only at this stage */
	ld	r5, THREAD_TM_DSCR(r3)
	ld	r6, THREAD_TM_PPR(r3)

	/* Clear the MSR RI since we are about to change R1.  EE is already off
	 */
	li	r4, 0
	mtmsrd	r4, 1

	REST_GPR(0, r7)				/* GPR0 */
	REST_2GPRS(2, r7)			/* GPR2-3 */
	REST_GPR(4, r7)				/* GPR4 */
	REST_4GPRS(8, r7)			/* GPR8-11 */
	REST_2GPRS(12, r7)			/* GPR12-13 */

	REST_NVGPRS(r7)				/* GPR14-31 */

	/* Load up PPR and DSCR here so we don't run with user values for long
	 */
	mtspr	SPRN_DSCR, r5
	mtspr	SPRN_PPR, r6

	/* Do final sanity check on TEXASR to make sure FS is set.  Do this
	 * here before we load up the userspace r1 so any bugs we hit will get
	 * a call chain */
	mfspr	r5, SPRN_TEXASR
	srdi	r5, r5, 16
	li	r6, (TEXASR_FS)@h
	and	r6, r6, r5
1:	tdeqi	r6, 0
	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0

	/* Do final sanity check on MSR to make sure we are not transactional
	 * or suspended
	 */
	mfmsr   r6
	li	r5, (MSR_TS_MASK)@higher
	srdi	r6, r6, 32
	and	r6, r6, r5
1:	tdnei   r6, 0
	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0

	/* Restore CR */
	ld	r6, _CCR(r7)
	mtcr    r6

	REST_GPR(1, r7)				/* GPR1 */
	REST_GPR(5, r7)				/* GPR5-7 */
	REST_GPR(6, r7)
	ld	r7, GPR7(r7)

	/* Commit register state as checkpointed state: */
	TRECHKPT

	HMT_MEDIUM

	/* Our transactional state has now changed.
	 *
	 * Now just get out of here.  Transactional (current) state will be
	 * updated once restore is called on the return path in the _switch-ed
	 * -to process.
	 */

	GET_PACA(r13)
	GET_SCRATCH0(r1)

	/* R1 is restored, so we are recoverable again.  EE is still off */
	li	r4, MSR_RI
	mtmsrd	r4, 1

	REST_NVGPRS(r1)

	addi    r1, r1, TM_FRAME_SIZE
	lwz	r4, 8(r1)
	ld	r0, 16(r1)
	mtcr	r4
	mtlr	r0
	ld	r2, STK_GOT(r1)

	/* Load CPU's default DSCR */
	ld	r0, PACA_DSCR_DEFAULT(r13)
	mtspr	SPRN_DSCR, r0

	blr

	/* ****************************************************************** */
Commit	Line	Data
98ae22e1 MN	1	/*
	2	* Transactional memory support routines to reclaim and recheckpoint
	3	* transactional process state.
	4	*
	5	* Copyright 2012 Matt Evans & Michael Neuling, IBM Corporation.
	6	*/
	7
	8	#include <asm/asm-offsets.h>
	9	#include <asm/ppc_asm.h>
	10	#include <asm/ppc-opcode.h>
	11	#include <asm/ptrace.h>
	12	#include <asm/reg.h>
7f06f21d	13	#include <asm/bug.h>
98ae22e1 MN	14
98ae22e1 MN	15	#ifdef CONFIG_VSX
de79f7b9 PM	16	/* See fpu.S, this is borrowed from there */
de79f7b9 PM	17	#define __SAVE_32FPRS_VSRS(n,c,base) \
98ae22e1 MN	18	BEGIN_FTR_SECTION \
	19	b 2f; \
	20	END_FTR_SECTION_IFSET(CPU_FTR_VSX); \
de79f7b9	21	SAVE_32FPRS(n,base); \
98ae22e1	22	b 3f; \
de79f7b9	23	2: SAVE_32VSRS(n,c,base); \
98ae22e1	24	3:
98ae22e1 MN	25	#define __REST_32FPRS_VSRS(n,c,base) \
	26	BEGIN_FTR_SECTION \
	27	b 2f; \
	28	END_FTR_SECTION_IFSET(CPU_FTR_VSX); \
	29	REST_32FPRS(n,base); \
	30	b 3f; \
	31	2: REST_32VSRS(n,c,base); \
	32	3:
	33	#else
de79f7b9 PM	34	#define __SAVE_32FPRS_VSRS(n,c,base) SAVE_32FPRS(n, base)
de79f7b9 PM	35	#define __REST_32FPRS_VSRS(n,c,base) REST_32FPRS(n, base)
98ae22e1	36	#endif
de79f7b9 PM	37	#define SAVE_32FPRS_VSRS(n,c,base) \
de79f7b9 PM	38	__SAVE_32FPRS_VSRS(n,__REG_##c,__REG_##base)
98ae22e1 MN	39	#define REST_32FPRS_VSRS(n,c,base) \
	40	__REST_32FPRS_VSRS(n,__REG_##c,__REG_##base)
	41
	42	/* Stack frame offsets for local variables. */
	43	#define TM_FRAME_L0 TM_FRAME_SIZE-16
	44	#define TM_FRAME_L1 TM_FRAME_SIZE-8
98ae22e1 MN	45
	46
	47	/* In order to access the TM SPRs, TM must be enabled. So, do so: */
	48	_GLOBAL(tm_enable)
	49	mfmsr r4
	50	li r3, MSR_TM >> 32
	51	sldi r3, r3, 32
	52	and. r0, r4, r3
	53	bne 1f
	54	or r4, r4, r3
	55	mtmsrd r4
	56	1: blr
	57
	58	_GLOBAL(tm_save_sprs)
	59	mfspr r0, SPRN_TFHAR
	60	std r0, THREAD_TM_TFHAR(r3)
	61	mfspr r0, SPRN_TEXASR
	62	std r0, THREAD_TM_TEXASR(r3)
	63	mfspr r0, SPRN_TFIAR
	64	std r0, THREAD_TM_TFIAR(r3)
	65	blr
	66
	67	_GLOBAL(tm_restore_sprs)
	68	ld r0, THREAD_TM_TFHAR(r3)
	69	mtspr SPRN_TFHAR, r0
	70	ld r0, THREAD_TM_TEXASR(r3)
	71	mtspr SPRN_TEXASR, r0
	72	ld r0, THREAD_TM_TFIAR(r3)
	73	mtspr SPRN_TFIAR, r0
	74	blr
	75
	76	/* Passed an 8-bit failure cause as first argument. */
	77	_GLOBAL(tm_abort)
	78	TABORT(R3)
	79	blr
	80
98ae22e1 MN	81	/* void tm_reclaim(struct thread_struct *thread,
	82	* unsigned long orig_msr,
	83	* uint8_t cause)
	84	*
	85	* - Performs a full reclaim. This destroys outstanding
	86	* transactions and updates thread->regs.tm_ckpt_* with the
	87	* original checkpointed state. Note that thread->regs is
	88	* unchanged.
	89	* - FP regs are written back to thread->transact_fpr before
	90	* reclaiming. These are the transactional (current) versions.
	91	*
	92	* Purpose is to both abort transactions of, and preserve the state of,
	93	* a transactions at a context switch. We preserve/restore both sets of process
	94	* state to restore them when the thread's scheduled again. We continue in
	95	* userland as though nothing happened, but when the transaction is resumed
	96	* they will abort back to the checkpointed state we save out here.
	97	*
	98	* Call with IRQs off, stacks get all out of sync for some periods in here!
	99	*/
	100	_GLOBAL(tm_reclaim)
	101	mfcr r6
	102	mflr r0
bbe30b3b	103	stw r6, 8(r1)
98ae22e1	104	std r0, 16(r1)
6403105b	105	std r2, STK_GOT(r1)
98ae22e1 MN	106	stdu r1, -TM_FRAME_SIZE(r1)
	107
	108	/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD]. */
	109
c2e31bdc	110	std r3, STK_PARAM(R3)(r1)
98ae22e1 MN	111	SAVE_NVGPRS(r1)
98ae22e1 MN	112
090b9284 MN	113	/* We need to setup MSR for VSX register save instructions. Here we
	114	* also clear the MSR RI since when we do the treclaim, we won't have a
	115	* valid kernel pointer for a while. We clear RI here as it avoids
	116	* adding another mtmsr closer to the treclaim. This makes the region
	117	* maked as non-recoverable wider than it needs to be but it saves on
	118	* inserting another mtmsrd later.
	119	*/
98ae22e1 MN	120	mfmsr r14
	121	mr r15, r14
	122	ori r15, r15, MSR_FP
090b9284	123	li r16, MSR_RI
c69e63b0	124	ori r16, r16, MSR_EE /* IRQs hard off */
090b9284	125	andc r15, r15, r16
98ae22e1 MN	126	oris r15, r15, MSR_VEC@h
	127	#ifdef CONFIG_VSX
	128	BEGIN_FTR_SECTION
	129	oris r15,r15, MSR_VSX@h
	130	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
	131	#endif
	132	mtmsrd r15
	133	std r14, TM_FRAME_L0(r1)
	134
	135	/* Stash the stack pointer away for use after reclaim */
	136	std r1, PACAR1(r13)
	137
	138	/* ****************** FPR/VR/VSRs **********
	139	* Before reclaiming, capture the current/transactional FPR/VR
	140	* versions /if used/.
	141	*
	142	* (If VSX used, FP and VMX are implied. Or, we don't need to look
	143	* at MSR.VSX as copying FP regs if .FP, vector regs if .VMX covers it.)
	144	*
	145	* We're passed the thread's MSR as parameter 2.
	146	*
	147	* We enabled VEC/FP/VSX in the msr above, so we can execute these
	148	* instructions!
	149	*/
	150	andis. r0, r4, MSR_VEC@h
	151	beq dont_backup_vec
	152
de79f7b9 PM	153	addi r7, r3, THREAD_TRANSACT_VRSTATE
de79f7b9 PM	154	SAVE_32VRS(0, r6, r7) /* r6 scratch, r7 transact vr state */
c2ce6f9f	155	mfvscr v0
de79f7b9	156	li r6, VRSTATE_VSCR
c2ce6f9f	157	stvx v0, r7, r6
408a7e08	158	dont_backup_vec:
98ae22e1 MN	159	mfspr r0, SPRN_VRSAVE
	160	std r0, THREAD_TRANSACT_VRSAVE(r3)
	161
98ae22e1 MN	162	andi. r0, r4, MSR_FP
	163	beq dont_backup_fp
	164
de79f7b9 PM	165	addi r7, r3, THREAD_TRANSACT_FPSTATE
de79f7b9 PM	166	SAVE_32FPRS_VSRS(0, R6, R7) /* r6 scratch, r7 transact fp state */
98ae22e1 MN	167
98ae22e1 MN	168	mffs fr0
de79f7b9	169	stfd fr0,FPSTATE_FPSCR(r7)
98ae22e1 MN	170
98ae22e1 MN	171	dont_backup_fp:
7f06f21d MN	172	/* Do sanity check on MSR to make sure we are suspended */
	173	li r7, (MSR_TS_S)@higher
	174	srdi r6, r14, 32
	175	and r6, r6, r7
	176	1: tdeqi r6, 0
	177	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
	178
98ae22e1 MN	179	/* The moment we treclaim, ALL of our GPRs will switch
	180	* to user register state. (FPRs, CCR etc. also!)
	181	* Use an sprg and a tm_scratch in the PACA to shuffle.
	182	*/
	183	TRECLAIM(R5) /* Cause in r5 */
	184
	185	/* ****************** GPRs ****************** */
	186	/* Stash the checkpointed r13 away in the scratch SPR and get the real
	187	* paca
	188	*/
	189	SET_SCRATCH0(r13)
	190	GET_PACA(r13)
	191
	192	/* Stash the checkpointed r1 away in paca tm_scratch and get the real
	193	* stack pointer back
	194	*/
	195	std r1, PACATMSCRATCH(r13)
	196	ld r1, PACAR1(r13)
	197
e9bdc3d6 MN	198	/* Store the PPR in r11 and reset to decent value */
	199	std r11, GPR11(r1) /* Temporary stash */
	200	mfspr r11, SPRN_PPR
	201	HMT_MEDIUM
	202
98ae22e1 MN	203	/* Now get some more GPRS free */
	204	std r7, GPR7(r1) /* Temporary stash */
	205	std r12, GPR12(r1) /* '' '' '' */
c2e31bdc	206	ld r12, STK_PARAM(R3)(r1) /* Param 0, thread_struct * */
98ae22e1	207
e9bdc3d6 MN	208	std r11, THREAD_TM_PPR(r12) /* Store PPR and free r11 */
e9bdc3d6 MN	209
98ae22e1 MN	210	addi r7, r12, PT_CKPT_REGS /* Thread's ckpt_regs */
	211
	212	/* Make r7 look like an exception frame so that we
	213	* can use the neat GPRx(n) macros. r7 is NOT a pt_regs ptr!
	214	*/
	215	subi r7, r7, STACK_FRAME_OVERHEAD
	216
	217	/* Sync the userland GPRs 2-12, 14-31 to thread->regs: */
	218	SAVE_GPR(0, r7) /* user r0 */
	219	SAVE_GPR(2, r7) /* user r2 */
	220	SAVE_4GPRS(3, r7) /* user r3-r6 */
e9bdc3d6 MN	221	SAVE_GPR(8, r7) /* user r8 */
	222	SAVE_GPR(9, r7) /* user r9 */
	223	SAVE_GPR(10, r7) /* user r10 */
98ae22e1 MN	224	ld r3, PACATMSCRATCH(r13) /* user r1 */
98ae22e1 MN	225	ld r4, GPR7(r1) /* user r7 */
e9bdc3d6 MN	226	ld r5, GPR11(r1) /* user r11 */
	227	ld r6, GPR12(r1) /* user r12 */
	228	GET_SCRATCH0(8) /* user r13 */
98ae22e1 MN	229	std r3, GPR1(r7)
98ae22e1 MN	230	std r4, GPR7(r7)
e9bdc3d6 MN	231	std r5, GPR11(r7)
	232	std r6, GPR12(r7)
	233	std r8, GPR13(r7)
98ae22e1 MN	234
	235	SAVE_NVGPRS(r7) /* user r14-r31 */
	236
	237	/* ****************** NIP ****************** */
	238	mfspr r3, SPRN_TFHAR
	239	std r3, _NIP(r7) /* Returns to failhandler */
	240	/* The checkpointed NIP is ignored when rescheduling/rechkpting,
	241	* but is used in signal return to 'wind back' to the abort handler.
	242	*/
	243
	244	/* ****************** CR,LR,CCR,MSR ******** */
	245	mfctr r3
	246	mflr r4
	247	mfcr r5
	248	mfxer r6
	249
	250	std r3, _CTR(r7)
	251	std r4, _LINK(r7)
	252	std r5, _CCR(r7)
	253	std r6, _XER(r7)
	254
28e61cc4	255
e9bdc3d6	256	/* ****************** TAR, DSCR ******** */
28e61cc4	257	mfspr r3, SPRN_TAR
e9bdc3d6	258	mfspr r4, SPRN_DSCR
28e61cc4 MN	259
28e61cc4 MN	260	std r3, THREAD_TM_TAR(r12)
e9bdc3d6	261	std r4, THREAD_TM_DSCR(r12)
28e61cc4	262
98ae22e1 MN	263	/* MSR and flags: We don't change CRs, and we don't need to alter
	264	* MSR.
	265	*/
	266
	267	/* TM regs, incl TEXASR -- these live in thread_struct. Note they've
	268	* been updated by the treclaim, to explain to userland the failure
	269	* cause (aborted).
	270	*/
	271	mfspr r0, SPRN_TEXASR
	272	mfspr r3, SPRN_TFHAR
	273	mfspr r4, SPRN_TFIAR
	274	std r0, THREAD_TM_TEXASR(r12)
	275	std r3, THREAD_TM_TFHAR(r12)
	276	std r4, THREAD_TM_TFIAR(r12)
	277
e9bdc3d6	278	/* AMR is checkpointed too, but is unsupported by Linux. */
98ae22e1 MN	279
	280	/* Restore original MSR/IRQ state & clear TM mode */
	281	ld r14, TM_FRAME_L0(r1) /* Orig MSR */
	282	li r15, 0
	283	rldimi r14, r15, MSR_TS_LG, (63-MSR_TS_LG)-1
	284	mtmsrd r14
	285
	286	REST_NVGPRS(r1)
	287
	288	addi r1, r1, TM_FRAME_SIZE
bbe30b3b	289	lwz r4, 8(r1)
98ae22e1 MN	290	ld r0, 16(r1)
	291	mtcr r4
	292	mtlr r0
6403105b	293	ld r2, STK_GOT(r1)
e9bdc3d6	294
1739ea9e	295	/* Load CPU's default DSCR */
1db36525	296	ld r0, PACA_DSCR_DEFAULT(r13)
e9bdc3d6 MN	297	mtspr SPRN_DSCR, r0
e9bdc3d6 MN	298
98ae22e1 MN	299	blr
	300
	301
	302	/* void tm_recheckpoint(struct thread_struct *thread,
	303	* unsigned long orig_msr)
	304	* - Restore the checkpointed register state saved by tm_reclaim
	305	* when we switch_to a process.
	306	*
	307	* Call with IRQs off, stacks get all out of sync for
	308	* some periods in here!
	309	*/
e6b8fd02	310	_GLOBAL(__tm_recheckpoint)
98ae22e1 MN	311	mfcr r5
98ae22e1 MN	312	mflr r0
bbe30b3b	313	stw r5, 8(r1)
98ae22e1	314	std r0, 16(r1)
6403105b	315	std r2, STK_GOT(r1)
98ae22e1 MN	316	stdu r1, -TM_FRAME_SIZE(r1)
	317
	318	/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD].
	319	* This is used for backing up the NVGPRs:
	320	*/
	321	SAVE_NVGPRS(r1)
	322
98ae22e1 MN	323	/* Load complete register state from ts_ckpt* registers */
	324
	325	addi r7, r3, PT_CKPT_REGS /* Thread's ckpt_regs */
	326
	327	/* Make r7 look like an exception frame so that we
	328	* can use the neat GPRx(n) macros. r7 is now NOT a pt_regs ptr!
	329	*/
	330	subi r7, r7, STACK_FRAME_OVERHEAD
	331
	332	SET_SCRATCH0(r1)
	333
	334	mfmsr r6
	335	/* R4 = original MSR to indicate whether thread used FP/Vector etc. */
	336
	337	/* Enable FP/vec in MSR if necessary! */
	338	lis r5, MSR_VEC@h
	339	ori r5, r5, MSR_FP
	340	and. r5, r4, r5
	341	beq restore_gprs /* if neither, skip both */
	342
	343	#ifdef CONFIG_VSX
	344	BEGIN_FTR_SECTION
	345	oris r5, r5, MSR_VSX@h
	346	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
	347	#endif
	348	or r5, r6, r5 /* Set MSR.FP+.VSX/.VEC */
	349	mtmsr r5
	350
f110c0c1	351	#ifdef CONFIG_ALTIVEC
98ae22e1 MN	352	/* FP and VEC registers: These are recheckpointed from thread.fpr[]
	353	* and thread.vr[] respectively. The thread.transact_fpr[] version
	354	* is more modern, and will be loaded subsequently by any FPUnavailable
	355	* trap.
	356	*/
	357	andis. r0, r4, MSR_VEC@h
	358	beq dont_restore_vec
	359
de79f7b9 PM	360	addi r8, r3, THREAD_VRSTATE
de79f7b9 PM	361	li r5, VRSTATE_VSCR
c2ce6f9f AB	362	lvx v0, r8, r5
c2ce6f9f AB	363	mtvscr v0
de79f7b9	364	REST_32VRS(0, r5, r8) /* r5 scratch, r8 ptr */
408a7e08	365	dont_restore_vec:
98ae22e1 MN	366	ld r5, THREAD_VRSAVE(r3)
98ae22e1 MN	367	mtspr SPRN_VRSAVE, r5
f110c0c1	368	#endif
98ae22e1	369
98ae22e1 MN	370	andi. r0, r4, MSR_FP
	371	beq dont_restore_fp
	372
de79f7b9 PM	373	addi r8, r3, THREAD_FPSTATE
de79f7b9 PM	374	lfd fr0, FPSTATE_FPSCR(r8)
98ae22e1	375	MTFSF_L(fr0)
de79f7b9	376	REST_32FPRS_VSRS(0, R4, R8)
98ae22e1 MN	377
	378	dont_restore_fp:
	379	mtmsr r6 /* FP/Vec off again! */
	380
	381	restore_gprs:
28e61cc4	382
e9bdc3d6 MN	383	/* ****************** CR,LR,CCR,MSR ******** */
	384	ld r4, _CTR(r7)
	385	ld r5, _LINK(r7)
e9bdc3d6	386	ld r8, _XER(r7)
28e61cc4	387
e9bdc3d6 MN	388	mtctr r4
e9bdc3d6 MN	389	mtlr r5
e9bdc3d6	390	mtxer r8
28e61cc4	391
e9bdc3d6 MN	392	/* ****************** TAR ****************** */
	393	ld r4, THREAD_TM_TAR(r3)
	394	mtspr SPRN_TAR, r4
98ae22e1	395
e9bdc3d6 MN	396	/* Load up the PPR and DSCR in GPRs only at this stage */
	397	ld r5, THREAD_TM_DSCR(r3)
	398	ld r6, THREAD_TM_PPR(r3)
98ae22e1	399
090b9284	400	/* Clear the MSR RI since we are about to change R1. EE is already off
98ae22e1	401	*/
090b9284 MN	402	li r4, 0
090b9284 MN	403	mtmsrd r4, 1
98ae22e1	404
7f06f21d MN	405	REST_GPR(0, r7) /* GPR0 */
7f06f21d MN	406	REST_2GPRS(2, r7) /* GPR2-3 */
e9bdc3d6	407	REST_GPR(4, r7) /* GPR4 */
98ae22e1 MN	408	REST_4GPRS(8, r7) /* GPR8-11 */
	409	REST_2GPRS(12, r7) /* GPR12-13 */
	410
	411	REST_NVGPRS(r7) /* GPR14-31 */
	412
e9bdc3d6 MN	413	/* Load up PPR and DSCR here so we don't run with user values for long
	414	*/
	415	mtspr SPRN_DSCR, r5
	416	mtspr SPRN_PPR, r6
	417
7f06f21d MN	418	/* Do final sanity check on TEXASR to make sure FS is set. Do this
	419	* here before we load up the userspace r1 so any bugs we hit will get
	420	* a call chain */
	421	mfspr r5, SPRN_TEXASR
	422	srdi r5, r5, 16
	423	li r6, (TEXASR_FS)@h
	424	and r6, r6, r5
	425	1: tdeqi r6, 0
	426	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
	427
	428	/* Do final sanity check on MSR to make sure we are not transactional
	429	* or suspended
	430	*/
	431	mfmsr r6
	432	li r5, (MSR_TS_MASK)@higher
	433	srdi r6, r6, 32
	434	and r6, r6, r5
	435	1: tdnei r6, 0
	436	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
	437
	438	/* Restore CR */
	439	ld r6, _CCR(r7)
	440	mtcr r6
	441
	442	REST_GPR(1, r7) /* GPR1 */
e9bdc3d6 MN	443	REST_GPR(5, r7) /* GPR5-7 */
	444	REST_GPR(6, r7)
	445	ld r7, GPR7(r7)
98ae22e1 MN	446
	447	/* Commit register state as checkpointed state: */
	448	TRECHKPT
	449
e9bdc3d6 MN	450	HMT_MEDIUM
e9bdc3d6 MN	451
98ae22e1 MN	452	/* Our transactional state has now changed.
	453	*
	454	* Now just get out of here. Transactional (current) state will be
	455	* updated once restore is called on the return path in the _switch-ed
	456	* -to process.
	457	*/
	458
	459	GET_PACA(r13)
	460	GET_SCRATCH0(r1)
	461
090b9284 MN	462	/* R1 is restored, so we are recoverable again. EE is still off */
	463	li r4, MSR_RI
	464	mtmsrd r4, 1
	465
98ae22e1 MN	466	REST_NVGPRS(r1)
	467
	468	addi r1, r1, TM_FRAME_SIZE
bbe30b3b	469	lwz r4, 8(r1)
98ae22e1 MN	470	ld r0, 16(r1)
	471	mtcr r4
	472	mtlr r0
6403105b	473	ld r2, STK_GOT(r1)
e9bdc3d6	474
1739ea9e	475	/* Load CPU's default DSCR */
1db36525	476	ld r0, PACA_DSCR_DEFAULT(r13)
e9bdc3d6 MN	477	mtspr SPRN_DSCR, r0
e9bdc3d6 MN	478
98ae22e1 MN	479	blr
	480
	481	/* ****************************************************************** */