[linux-block.git] / arch / powerpc / kernel / tm.S

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Transactional memory support routines to reclaim and recheckpoint
 * transactional process state.
 *
 * Copyright 2012 Matt Evans & Michael Neuling, IBM Corporation.
 */

#include <asm/asm-offsets.h>
#include <asm/ppc_asm.h>
#include <asm/ppc-opcode.h>
#include <asm/ptrace.h>
#include <asm/reg.h>
#include <asm/bug.h>
#include <asm/export.h>
#include <asm/feature-fixups.h>

#ifdef CONFIG_VSX
/* See fpu.S, this is borrowed from there */
#define __SAVE_32FPRS_VSRS(n,c,base)		\
BEGIN_FTR_SECTION				\
	b	2f;				\
END_FTR_SECTION_IFSET(CPU_FTR_VSX);		\
	SAVE_32FPRS(n,base);			\
	b	3f;				\
2:	SAVE_32VSRS(n,c,base);			\
3:
#define __REST_32FPRS_VSRS(n,c,base)		\
BEGIN_FTR_SECTION				\
	b	2f;				\
END_FTR_SECTION_IFSET(CPU_FTR_VSX);		\
	REST_32FPRS(n,base);			\
	b	3f;				\
2:	REST_32VSRS(n,c,base);			\
3:
#else
#define __SAVE_32FPRS_VSRS(n,c,base)	SAVE_32FPRS(n, base)
#define __REST_32FPRS_VSRS(n,c,base)	REST_32FPRS(n, base)
#endif
#define SAVE_32FPRS_VSRS(n,c,base) \
	__SAVE_32FPRS_VSRS(n,__REG_##c,__REG_##base)
#define REST_32FPRS_VSRS(n,c,base) \
	__REST_32FPRS_VSRS(n,__REG_##c,__REG_##base)

/* Stack frame offsets for local variables. */
#define TM_FRAME_L0	TM_FRAME_SIZE-16
#define TM_FRAME_L1	TM_FRAME_SIZE-8


/* In order to access the TM SPRs, TM must be enabled.  So, do so: */
_GLOBAL(tm_enable)
	mfmsr	r4
	li	r3, MSR_TM >> 32
	sldi	r3, r3, 32
	and.	r0, r4, r3
	bne	1f
	or	r4, r4, r3
	mtmsrd	r4
1:	blr
EXPORT_SYMBOL_GPL(tm_enable);

_GLOBAL(tm_disable)
	mfmsr	r4
	li	r3, MSR_TM >> 32
	sldi	r3, r3, 32
	andc	r4, r4, r3
	mtmsrd	r4
	blr
EXPORT_SYMBOL_GPL(tm_disable);

_GLOBAL(tm_save_sprs)
	mfspr	r0, SPRN_TFHAR
	std	r0, THREAD_TM_TFHAR(r3)
	mfspr	r0, SPRN_TEXASR
	std	r0, THREAD_TM_TEXASR(r3)
	mfspr	r0, SPRN_TFIAR
	std	r0, THREAD_TM_TFIAR(r3)
	blr

_GLOBAL(tm_restore_sprs)
	ld	r0, THREAD_TM_TFHAR(r3)
	mtspr	SPRN_TFHAR, r0
	ld	r0, THREAD_TM_TEXASR(r3)
	mtspr	SPRN_TEXASR, r0
	ld	r0, THREAD_TM_TFIAR(r3)
	mtspr	SPRN_TFIAR, r0
	blr

	/* Passed an 8-bit failure cause as first argument. */
_GLOBAL(tm_abort)
	TABORT(R3)
	blr
EXPORT_SYMBOL_GPL(tm_abort);

/* void tm_reclaim(struct thread_struct *thread,
 *		   uint8_t cause)
 *
 *	- Performs a full reclaim.  This destroys outstanding
 *	  transactions and updates thread->regs.tm_ckpt_* with the
 *	  original checkpointed state.  Note that thread->regs is
 *	  unchanged.
 *
 * Purpose is to both abort transactions of, and preserve the state of,
 * a transactions at a context switch. We preserve/restore both sets of process
 * state to restore them when the thread's scheduled again.  We continue in
 * userland as though nothing happened, but when the transaction is resumed
 * they will abort back to the checkpointed state we save out here.
 *
 * Call with IRQs off, stacks get all out of sync for some periods in here!
 */
_GLOBAL(tm_reclaim)
	mfcr	r5
	mflr	r0
	stw	r5, 8(r1)
	std	r0, 16(r1)
	std	r2, STK_GOT(r1)
	stdu	r1, -TM_FRAME_SIZE(r1)

	/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD]. */

	std	r3, STK_PARAM(R3)(r1)
	SAVE_NVGPRS(r1)

	/* We need to setup MSR for VSX register save instructions. */
	mfmsr	r14
	mr	r15, r14
	ori	r15, r15, MSR_FP
	li	r16, 0
	ori	r16, r16, MSR_EE /* IRQs hard off */
	andc	r15, r15, r16
	oris	r15, r15, MSR_VEC@h
#ifdef CONFIG_VSX
	BEGIN_FTR_SECTION
	oris	r15,r15, MSR_VSX@h
	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	mtmsrd	r15
	std	r14, TM_FRAME_L0(r1)

	/* Do sanity check on MSR to make sure we are suspended */
	li	r7, (MSR_TS_S)@higher
	srdi	r6, r14, 32
	and	r6, r6, r7
1:	tdeqi   r6, 0
	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0

	/* Stash the stack pointer away for use after reclaim */
	std	r1, PACAR1(r13)

	/* Clear MSR RI since we are about to change r1, EE is already off. */
	li	r5, 0
	mtmsrd	r5, 1

	/*
	 * BE CAREFUL HERE:
	 * At this point we can't take an SLB miss since we have MSR_RI
	 * off. Load only to/from the stack/paca which are in SLB bolted regions
	 * until we turn MSR RI back on.
	 *
	 * The moment we treclaim, ALL of our GPRs will switch
	 * to user register state.  (FPRs, CCR etc. also!)
	 * Use an sprg and a tm_scratch in the PACA to shuffle.
	 */
	TRECLAIM(R4)				/* Cause in r4 */

	/* ******************** GPRs ******************** */
	/* Stash the checkpointed r13 away in the scratch SPR and get the real
	 *  paca
	 */
	SET_SCRATCH0(r13)
	GET_PACA(r13)

	/* Stash the checkpointed r1 away in paca tm_scratch and get the real
	 * stack pointer back
	 */
	std	r1, PACATMSCRATCH(r13)
	ld	r1, PACAR1(r13)

	std	r11, GPR11(r1)			/* Temporary stash */

	/*
	 * Move the saved user r1 to the kernel stack in case PACATMSCRATCH is
	 * clobbered by an exception once we turn on MSR_RI below.
	 */
	ld	r11, PACATMSCRATCH(r13)
	std	r11, GPR1(r1)

	/*
	 * Store r13 away so we can free up the scratch SPR for the SLB fault
	 * handler (needed once we start accessing the thread_struct).
	 */
	GET_SCRATCH0(r11)
	std	r11, GPR13(r1)

	/* Reset MSR RI so we can take SLB faults again */
	li	r11, MSR_RI
	mtmsrd	r11, 1

	/* Store the PPR in r11 and reset to decent value */
	mfspr	r11, SPRN_PPR
	HMT_MEDIUM

	/* Now get some more GPRS free */
	std	r7, GPR7(r1)			/* Temporary stash */
	std	r12, GPR12(r1)			/* ''   ''    ''   */
	ld	r12, STK_PARAM(R3)(r1)		/* Param 0, thread_struct * */

	std	r11, THREAD_TM_PPR(r12)		/* Store PPR and free r11 */

	addi	r7, r12, PT_CKPT_REGS		/* Thread's ckpt_regs */

	/* Make r7 look like an exception frame so that we
	 * can use the neat GPRx(n) macros.  r7 is NOT a pt_regs ptr!
	 */
	subi	r7, r7, STACK_FRAME_OVERHEAD

	/* Sync the userland GPRs 2-12, 14-31 to thread->regs: */
	SAVE_GPR(0, r7)				/* user r0 */
	SAVE_GPR(2, r7)			/* user r2 */
	SAVE_4GPRS(3, r7)			/* user r3-r6 */
	SAVE_GPR(8, r7)				/* user r8 */
	SAVE_GPR(9, r7)				/* user r9 */
	SAVE_GPR(10, r7)			/* user r10 */
	ld	r3, GPR1(r1)			/* user r1 */
	ld	r4, GPR7(r1)			/* user r7 */
	ld	r5, GPR11(r1)			/* user r11 */
	ld	r6, GPR12(r1)			/* user r12 */
	ld	r8, GPR13(r1)			/* user r13 */
	std	r3, GPR1(r7)
	std	r4, GPR7(r7)
	std	r5, GPR11(r7)
	std	r6, GPR12(r7)
	std	r8, GPR13(r7)

	SAVE_NVGPRS(r7)				/* user r14-r31 */

	/* ******************** NIP ******************** */
	mfspr	r3, SPRN_TFHAR
	std	r3, _NIP(r7)			/* Returns to failhandler */
	/* The checkpointed NIP is ignored when rescheduling/rechkpting,
	 * but is used in signal return to 'wind back' to the abort handler.
	 */

	/* ******************** CR,LR,CCR,MSR ********** */
	mfctr	r3
	mflr	r4
	mfcr	r5
	mfxer	r6

	std	r3, _CTR(r7)
	std	r4, _LINK(r7)
	std	r5, _CCR(r7)
	std	r6, _XER(r7)


	/* ******************** TAR, DSCR ********** */
	mfspr	r3, SPRN_TAR
	mfspr	r4, SPRN_DSCR

	std	r3, THREAD_TM_TAR(r12)
	std	r4, THREAD_TM_DSCR(r12)

	/* MSR and flags:  We don't change CRs, and we don't need to alter
	 * MSR.
	 */


	/* ******************** FPR/VR/VSRs ************
	 * After reclaiming, capture the checkpointed FPRs/VRs.
	 *
	 * We enabled VEC/FP/VSX in the msr above, so we can execute these
	 * instructions!
	 */
	mr	r3, r12

	/* Altivec (VEC/VMX/VR)*/
	addi	r7, r3, THREAD_CKVRSTATE
	SAVE_32VRS(0, r6, r7)	/* r6 scratch, r7 transact vr state */
	mfvscr	v0
	li	r6, VRSTATE_VSCR
	stvx	v0, r7, r6

	/* VRSAVE */
	mfspr	r0, SPRN_VRSAVE
	std	r0, THREAD_CKVRSAVE(r3)

	/* Floating Point (FP) */
	addi	r7, r3, THREAD_CKFPSTATE
	SAVE_32FPRS_VSRS(0, R6, R7)	/* r6 scratch, r7 transact fp state */
	mffs    fr0
	stfd    fr0,FPSTATE_FPSCR(r7)


	/* TM regs, incl TEXASR -- these live in thread_struct.  Note they've
	 * been updated by the treclaim, to explain to userland the failure
	 * cause (aborted).
	 */
	mfspr	r0, SPRN_TEXASR
	mfspr	r3, SPRN_TFHAR
	mfspr	r4, SPRN_TFIAR
	std	r0, THREAD_TM_TEXASR(r12)
	std	r3, THREAD_TM_TFHAR(r12)
	std	r4, THREAD_TM_TFIAR(r12)

	/* AMR is checkpointed too, but is unsupported by Linux. */

	/* Restore original MSR/IRQ state & clear TM mode */
	ld	r14, TM_FRAME_L0(r1)		/* Orig MSR */

	li	r15, 0
	rldimi  r14, r15, MSR_TS_LG, (63-MSR_TS_LG)-1
	mtmsrd  r14

	REST_NVGPRS(r1)

	addi    r1, r1, TM_FRAME_SIZE
	lwz	r4, 8(r1)
	ld	r0, 16(r1)
	mtcr	r4
	mtlr	r0
	ld	r2, STK_GOT(r1)

	/* Load CPU's default DSCR */
	ld	r0, PACA_DSCR_DEFAULT(r13)
	mtspr	SPRN_DSCR, r0

	blr


       /*
	 * void __tm_recheckpoint(struct thread_struct *thread)
	 *	- Restore the checkpointed register state saved by tm_reclaim
	 *	  when we switch_to a process.
	 *
	 *	Call with IRQs off, stacks get all out of sync for
	 *	some periods in here!
	 */
_GLOBAL(__tm_recheckpoint)
	mfcr	r5
	mflr	r0
	stw	r5, 8(r1)
	std	r0, 16(r1)
	std	r2, STK_GOT(r1)
	stdu	r1, -TM_FRAME_SIZE(r1)

	/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD].
	 * This is used for backing up the NVGPRs:
	 */
	SAVE_NVGPRS(r1)

	/* Load complete register state from ts_ckpt* registers */

	addi	r7, r3, PT_CKPT_REGS		/* Thread's ckpt_regs */

	/* Make r7 look like an exception frame so that we
	 * can use the neat GPRx(n) macros.  r7 is now NOT a pt_regs ptr!
	 */
	subi	r7, r7, STACK_FRAME_OVERHEAD

	/* We need to setup MSR for FP/VMX/VSX register save instructions. */
	mfmsr	r6
	mr	r5, r6
	ori	r5, r5, MSR_FP
#ifdef CONFIG_ALTIVEC
	oris	r5, r5, MSR_VEC@h
#endif
#ifdef CONFIG_VSX
	BEGIN_FTR_SECTION
	oris	r5,r5, MSR_VSX@h
	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	mtmsrd	r5

#ifdef CONFIG_ALTIVEC
	/*
	 * FP and VEC registers: These are recheckpointed from
	 * thread.ckfp_state and thread.ckvr_state respectively. The
	 * thread.fp_state[] version holds the 'live' (transactional)
	 * and will be loaded subsequently by any FPUnavailable trap.
	 */
	addi	r8, r3, THREAD_CKVRSTATE
	li	r5, VRSTATE_VSCR
	lvx	v0, r8, r5
	mtvscr	v0
	REST_32VRS(0, r5, r8)			/* r5 scratch, r8 ptr */
	ld	r5, THREAD_CKVRSAVE(r3)
	mtspr	SPRN_VRSAVE, r5
#endif

	addi	r8, r3, THREAD_CKFPSTATE
	lfd	fr0, FPSTATE_FPSCR(r8)
	MTFSF_L(fr0)
	REST_32FPRS_VSRS(0, R4, R8)

	mtmsr	r6				/* FP/Vec off again! */

restore_gprs:

	/* ******************** CR,LR,CCR,MSR ********** */
	ld	r4, _CTR(r7)
	ld	r5, _LINK(r7)
	ld	r8, _XER(r7)

	mtctr	r4
	mtlr	r5
	mtxer	r8

	/* ******************** TAR ******************** */
	ld	r4, THREAD_TM_TAR(r3)
	mtspr	SPRN_TAR,	r4

	/* Load up the PPR and DSCR in GPRs only at this stage */
	ld	r5, THREAD_TM_DSCR(r3)
	ld	r6, THREAD_TM_PPR(r3)

	REST_GPR(0, r7)				/* GPR0 */
	REST_2GPRS(2, r7)			/* GPR2-3 */
	REST_GPR(4, r7)				/* GPR4 */
	REST_4GPRS(8, r7)			/* GPR8-11 */
	REST_2GPRS(12, r7)			/* GPR12-13 */

	REST_NVGPRS(r7)				/* GPR14-31 */

	/* Load up PPR and DSCR here so we don't run with user values for long
	 */
	mtspr	SPRN_DSCR, r5
	mtspr	SPRN_PPR, r6

	/* Do final sanity check on TEXASR to make sure FS is set.  Do this
	 * here before we load up the userspace r1 so any bugs we hit will get
	 * a call chain */
	mfspr	r5, SPRN_TEXASR
	srdi	r5, r5, 16
	li	r6, (TEXASR_FS)@h
	and	r6, r6, r5
1:	tdeqi	r6, 0
	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0

	/* Do final sanity check on MSR to make sure we are not transactional
	 * or suspended
	 */
	mfmsr   r6
	li	r5, (MSR_TS_MASK)@higher
	srdi	r6, r6, 32
	and	r6, r6, r5
1:	tdnei   r6, 0
	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0

	/* Restore CR */
	ld	r6, _CCR(r7)
	mtcr    r6

	REST_GPR(6, r7)

	/*
	 * Store r1 and r5 on the stack so that we can access them
	 * after we clear MSR RI.
	 */

	REST_GPR(5, r7)
	std	r5, -8(r1)
	ld	r5, GPR1(r7)
	std	r5, -16(r1)

	REST_GPR(7, r7)

	/* Clear MSR RI since we are about to change r1. EE is already off */
	li	r5, 0
	mtmsrd	r5, 1

	/*
	 * BE CAREFUL HERE:
	 * At this point we can't take an SLB miss since we have MSR_RI
	 * off. Load only to/from the stack/paca which are in SLB bolted regions
	 * until we turn MSR RI back on.
	 */

	SET_SCRATCH0(r1)
	ld	r5, -8(r1)
	ld	r1, -16(r1)

	/* Commit register state as checkpointed state: */
	TRECHKPT

	HMT_MEDIUM

	/* Our transactional state has now changed.
	 *
	 * Now just get out of here.  Transactional (current) state will be
	 * updated once restore is called on the return path in the _switch-ed
	 * -to process.
	 */

	GET_PACA(r13)
	GET_SCRATCH0(r1)

	/* R1 is restored, so we are recoverable again.  EE is still off */
	li	r4, MSR_RI
	mtmsrd	r4, 1

	REST_NVGPRS(r1)

	addi    r1, r1, TM_FRAME_SIZE
	lwz	r4, 8(r1)
	ld	r0, 16(r1)
	mtcr	r4
	mtlr	r0
	ld	r2, STK_GOT(r1)

	/* Load CPU's default DSCR */
	ld	r0, PACA_DSCR_DEFAULT(r13)
	mtspr	SPRN_DSCR, r0

	blr

	/* ****************************************************************** */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
98ae22e1 MN	2	/*
	3	* Transactional memory support routines to reclaim and recheckpoint
	4	* transactional process state.
	5	*
	6	* Copyright 2012 Matt Evans & Michael Neuling, IBM Corporation.
	7	*/
	8
	9	#include <asm/asm-offsets.h>
	10	#include <asm/ppc_asm.h>
	11	#include <asm/ppc-opcode.h>
	12	#include <asm/ptrace.h>
	13	#include <asm/reg.h>
7f06f21d	14	#include <asm/bug.h>
eacbb218	15	#include <asm/export.h>
2c86cd18	16	#include <asm/feature-fixups.h>
98ae22e1 MN	17
98ae22e1 MN	18	#ifdef CONFIG_VSX
de79f7b9 PM	19	/* See fpu.S, this is borrowed from there */
de79f7b9 PM	20	#define __SAVE_32FPRS_VSRS(n,c,base) \
98ae22e1 MN	21	BEGIN_FTR_SECTION \
	22	b 2f; \
	23	END_FTR_SECTION_IFSET(CPU_FTR_VSX); \
de79f7b9	24	SAVE_32FPRS(n,base); \
98ae22e1	25	b 3f; \
de79f7b9	26	2: SAVE_32VSRS(n,c,base); \
98ae22e1	27	3:
98ae22e1 MN	28	#define __REST_32FPRS_VSRS(n,c,base) \
	29	BEGIN_FTR_SECTION \
	30	b 2f; \
	31	END_FTR_SECTION_IFSET(CPU_FTR_VSX); \
	32	REST_32FPRS(n,base); \
	33	b 3f; \
	34	2: REST_32VSRS(n,c,base); \
	35	3:
	36	#else
de79f7b9 PM	37	#define __SAVE_32FPRS_VSRS(n,c,base) SAVE_32FPRS(n, base)
de79f7b9 PM	38	#define __REST_32FPRS_VSRS(n,c,base) REST_32FPRS(n, base)
98ae22e1	39	#endif
de79f7b9 PM	40	#define SAVE_32FPRS_VSRS(n,c,base) \
de79f7b9 PM	41	__SAVE_32FPRS_VSRS(n,__REG_##c,__REG_##base)
98ae22e1 MN	42	#define REST_32FPRS_VSRS(n,c,base) \
	43	__REST_32FPRS_VSRS(n,__REG_##c,__REG_##base)
	44
	45	/* Stack frame offsets for local variables. */
	46	#define TM_FRAME_L0 TM_FRAME_SIZE-16
	47	#define TM_FRAME_L1 TM_FRAME_SIZE-8
98ae22e1 MN	48
	49
	50	/* In order to access the TM SPRs, TM must be enabled. So, do so: */
	51	_GLOBAL(tm_enable)
	52	mfmsr r4
	53	li r3, MSR_TM >> 32
	54	sldi r3, r3, 32
	55	and. r0, r4, r3
	56	bne 1f
	57	or r4, r4, r3
	58	mtmsrd r4
	59	1: blr
eacbb218 SG	60	EXPORT_SYMBOL_GPL(tm_enable);
	61
	62	_GLOBAL(tm_disable)
	63	mfmsr r4
	64	li r3, MSR_TM >> 32
	65	sldi r3, r3, 32
	66	andc r4, r4, r3
	67	mtmsrd r4
	68	blr
	69	EXPORT_SYMBOL_GPL(tm_disable);
98ae22e1 MN	70
	71	_GLOBAL(tm_save_sprs)
	72	mfspr r0, SPRN_TFHAR
	73	std r0, THREAD_TM_TFHAR(r3)
	74	mfspr r0, SPRN_TEXASR
	75	std r0, THREAD_TM_TEXASR(r3)
	76	mfspr r0, SPRN_TFIAR
	77	std r0, THREAD_TM_TFIAR(r3)
	78	blr
	79
	80	_GLOBAL(tm_restore_sprs)
	81	ld r0, THREAD_TM_TFHAR(r3)
	82	mtspr SPRN_TFHAR, r0
	83	ld r0, THREAD_TM_TEXASR(r3)
	84	mtspr SPRN_TEXASR, r0
	85	ld r0, THREAD_TM_TFIAR(r3)
	86	mtspr SPRN_TFIAR, r0
	87	blr
	88
	89	/* Passed an 8-bit failure cause as first argument. */
	90	_GLOBAL(tm_abort)
	91	TABORT(R3)
	92	blr
eacbb218	93	EXPORT_SYMBOL_GPL(tm_abort);
98ae22e1	94
98ae22e1	95	/* void tm_reclaim(struct thread_struct *thread,
98ae22e1 MN	96	* uint8_t cause)
	97	*
	98	* - Performs a full reclaim. This destroys outstanding
	99	* transactions and updates thread->regs.tm_ckpt_* with the
	100	* original checkpointed state. Note that thread->regs is
	101	* unchanged.
98ae22e1 MN	102	*
	103	* Purpose is to both abort transactions of, and preserve the state of,
	104	* a transactions at a context switch. We preserve/restore both sets of process
	105	* state to restore them when the thread's scheduled again. We continue in
	106	* userland as though nothing happened, but when the transaction is resumed
	107	* they will abort back to the checkpointed state we save out here.
	108	*
	109	* Call with IRQs off, stacks get all out of sync for some periods in here!
	110	*/
	111	_GLOBAL(tm_reclaim)
eb5c3f1c	112	mfcr r5
98ae22e1	113	mflr r0
eb5c3f1c	114	stw r5, 8(r1)
98ae22e1	115	std r0, 16(r1)
6403105b	116	std r2, STK_GOT(r1)
98ae22e1 MN	117	stdu r1, -TM_FRAME_SIZE(r1)
	118
	119	/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD]. */
	120
c2e31bdc	121	std r3, STK_PARAM(R3)(r1)
98ae22e1 MN	122	SAVE_NVGPRS(r1)
98ae22e1 MN	123
190ce869	124	/* We need to setup MSR for VSX register save instructions. */
98ae22e1 MN	125	mfmsr r14
	126	mr r15, r14
	127	ori r15, r15, MSR_FP
190ce869	128	li r16, 0
c69e63b0	129	ori r16, r16, MSR_EE /* IRQs hard off */
090b9284	130	andc r15, r15, r16
98ae22e1 MN	131	oris r15, r15, MSR_VEC@h
	132	#ifdef CONFIG_VSX
	133	BEGIN_FTR_SECTION
	134	oris r15,r15, MSR_VSX@h
	135	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
	136	#endif
	137	mtmsrd r15
	138	std r14, TM_FRAME_L0(r1)
	139
7f06f21d MN	140	/* Do sanity check on MSR to make sure we are suspended */
	141	li r7, (MSR_TS_S)@higher
	142	srdi r6, r14, 32
	143	and r6, r6, r7
	144	1: tdeqi r6, 0
	145	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
	146
dc310669 CB	147	/* Stash the stack pointer away for use after reclaim */
	148	std r1, PACAR1(r13)
	149
190ce869	150	/* Clear MSR RI since we are about to change r1, EE is already off. */
eb5c3f1c CB	151	li r5, 0
eb5c3f1c CB	152	mtmsrd r5, 1
190ce869 MN	153
	154	/*
	155	* BE CAREFUL HERE:
	156	* At this point we can't take an SLB miss since we have MSR_RI
	157	* off. Load only to/from the stack/paca which are in SLB bolted regions
	158	* until we turn MSR RI back on.
	159	*
	160	* The moment we treclaim, ALL of our GPRs will switch
98ae22e1 MN	161	* to user register state. (FPRs, CCR etc. also!)
	162	* Use an sprg and a tm_scratch in the PACA to shuffle.
	163	*/
eb5c3f1c	164	TRECLAIM(R4) /* Cause in r4 */
98ae22e1 MN	165
	166	/* ****************** GPRs ****************** */
	167	/* Stash the checkpointed r13 away in the scratch SPR and get the real
	168	* paca
	169	*/
	170	SET_SCRATCH0(r13)
	171	GET_PACA(r13)
	172
	173	/* Stash the checkpointed r1 away in paca tm_scratch and get the real
	174	* stack pointer back
	175	*/
	176	std r1, PACATMSCRATCH(r13)
	177	ld r1, PACAR1(r13)
	178
e9bdc3d6	179	std r11, GPR11(r1) /* Temporary stash */
190ce869	180
96dc89d5 MN	181	/*
	182	* Move the saved user r1 to the kernel stack in case PACATMSCRATCH is
	183	* clobbered by an exception once we turn on MSR_RI below.
	184	*/
	185	ld r11, PACATMSCRATCH(r13)
	186	std r11, GPR1(r1)
	187
cf13435b MN	188	/*
	189	* Store r13 away so we can free up the scratch SPR for the SLB fault
	190	* handler (needed once we start accessing the thread_struct).
	191	*/
	192	GET_SCRATCH0(r11)
	193	std r11, GPR13(r1)
	194
190ce869 MN	195	/* Reset MSR RI so we can take SLB faults again */
	196	li r11, MSR_RI
	197	mtmsrd r11, 1
	198
cf13435b	199	/* Store the PPR in r11 and reset to decent value */
e9bdc3d6 MN	200	mfspr r11, SPRN_PPR
	201	HMT_MEDIUM
	202
98ae22e1 MN	203	/* Now get some more GPRS free */
	204	std r7, GPR7(r1) /* Temporary stash */
	205	std r12, GPR12(r1) /* '' '' '' */
c2e31bdc	206	ld r12, STK_PARAM(R3)(r1) /* Param 0, thread_struct * */
98ae22e1	207
e9bdc3d6 MN	208	std r11, THREAD_TM_PPR(r12) /* Store PPR and free r11 */
e9bdc3d6 MN	209
98ae22e1 MN	210	addi r7, r12, PT_CKPT_REGS /* Thread's ckpt_regs */
	211
	212	/* Make r7 look like an exception frame so that we
	213	* can use the neat GPRx(n) macros. r7 is NOT a pt_regs ptr!
	214	*/
	215	subi r7, r7, STACK_FRAME_OVERHEAD
	216
	217	/* Sync the userland GPRs 2-12, 14-31 to thread->regs: */
	218	SAVE_GPR(0, r7) /* user r0 */
	219	SAVE_GPR(2, r7) /* user r2 */
	220	SAVE_4GPRS(3, r7) /* user r3-r6 */
e9bdc3d6 MN	221	SAVE_GPR(8, r7) /* user r8 */
	222	SAVE_GPR(9, r7) /* user r9 */
	223	SAVE_GPR(10, r7) /* user r10 */
96dc89d5	224	ld r3, GPR1(r1) /* user r1 */
98ae22e1	225	ld r4, GPR7(r1) /* user r7 */
e9bdc3d6 MN	226	ld r5, GPR11(r1) /* user r11 */
e9bdc3d6 MN	227	ld r6, GPR12(r1) /* user r12 */
cf13435b	228	ld r8, GPR13(r1) /* user r13 */
98ae22e1 MN	229	std r3, GPR1(r7)
98ae22e1 MN	230	std r4, GPR7(r7)
e9bdc3d6 MN	231	std r5, GPR11(r7)
	232	std r6, GPR12(r7)
	233	std r8, GPR13(r7)
98ae22e1 MN	234
	235	SAVE_NVGPRS(r7) /* user r14-r31 */
	236
	237	/* ****************** NIP ****************** */
	238	mfspr r3, SPRN_TFHAR
	239	std r3, _NIP(r7) /* Returns to failhandler */
	240	/* The checkpointed NIP is ignored when rescheduling/rechkpting,
	241	* but is used in signal return to 'wind back' to the abort handler.
	242	*/
	243
	244	/* ****************** CR,LR,CCR,MSR ******** */
	245	mfctr r3
	246	mflr r4
	247	mfcr r5
	248	mfxer r6
	249
	250	std r3, _CTR(r7)
	251	std r4, _LINK(r7)
	252	std r5, _CCR(r7)
	253	std r6, _XER(r7)
	254
28e61cc4	255
e9bdc3d6	256	/* ****************** TAR, DSCR ******** */
28e61cc4	257	mfspr r3, SPRN_TAR
e9bdc3d6	258	mfspr r4, SPRN_DSCR
28e61cc4 MN	259
28e61cc4 MN	260	std r3, THREAD_TM_TAR(r12)
e9bdc3d6	261	std r4, THREAD_TM_DSCR(r12)
28e61cc4	262
98ae22e1 MN	263	/* MSR and flags: We don't change CRs, and we don't need to alter
	264	* MSR.
	265	*/
	266
dc310669 CB	267
dc310669 CB	268	/* ****************** FPR/VR/VSRs **********
eb5c3f1c	269	* After reclaiming, capture the checkpointed FPRs/VRs.
dc310669 CB	270	*
	271	* We enabled VEC/FP/VSX in the msr above, so we can execute these
	272	* instructions!
	273	*/
dc310669	274	mr r3, r12
dc310669	275
eb5c3f1c	276	/* Altivec (VEC/VMX/VR)*/
000ec280	277	addi r7, r3, THREAD_CKVRSTATE
dc310669 CB	278	SAVE_32VRS(0, r6, r7) /* r6 scratch, r7 transact vr state */
	279	mfvscr v0
	280	li r6, VRSTATE_VSCR
	281	stvx v0, r7, r6
eb5c3f1c CB	282
eb5c3f1c CB	283	/* VRSAVE */
dc310669	284	mfspr r0, SPRN_VRSAVE
000ec280	285	std r0, THREAD_CKVRSAVE(r3)
dc310669	286
eb5c3f1c	287	/* Floating Point (FP) */
000ec280	288	addi r7, r3, THREAD_CKFPSTATE
dc310669	289	SAVE_32FPRS_VSRS(0, R6, R7) /* r6 scratch, r7 transact fp state */
dc310669 CB	290	mffs fr0
	291	stfd fr0,FPSTATE_FPSCR(r7)
	292
dc310669	293
98ae22e1 MN	294	/* TM regs, incl TEXASR -- these live in thread_struct. Note they've
	295	* been updated by the treclaim, to explain to userland the failure
	296	* cause (aborted).
	297	*/
	298	mfspr r0, SPRN_TEXASR
	299	mfspr r3, SPRN_TFHAR
	300	mfspr r4, SPRN_TFIAR
	301	std r0, THREAD_TM_TEXASR(r12)
	302	std r3, THREAD_TM_TFHAR(r12)
	303	std r4, THREAD_TM_TFIAR(r12)
	304
e9bdc3d6	305	/* AMR is checkpointed too, but is unsupported by Linux. */
98ae22e1 MN	306
	307	/* Restore original MSR/IRQ state & clear TM mode */
	308	ld r14, TM_FRAME_L0(r1) /* Orig MSR */
dc310669	309
98ae22e1 MN	310	li r15, 0
	311	rldimi r14, r15, MSR_TS_LG, (63-MSR_TS_LG)-1
	312	mtmsrd r14
	313
	314	REST_NVGPRS(r1)
	315
	316	addi r1, r1, TM_FRAME_SIZE
bbe30b3b	317	lwz r4, 8(r1)
98ae22e1 MN	318	ld r0, 16(r1)
	319	mtcr r4
	320	mtlr r0
6403105b	321	ld r2, STK_GOT(r1)
e9bdc3d6	322
1739ea9e	323	/* Load CPU's default DSCR */
1db36525	324	ld r0, PACA_DSCR_DEFAULT(r13)
e9bdc3d6 MN	325	mtspr SPRN_DSCR, r0
e9bdc3d6 MN	326
98ae22e1 MN	327	blr
	328
	329
a596a7e9 CB	330	/*
a596a7e9 CB	331	* void __tm_recheckpoint(struct thread_struct *thread)
98ae22e1 MN	332	* - Restore the checkpointed register state saved by tm_reclaim
	333	* when we switch_to a process.
	334	*
	335	* Call with IRQs off, stacks get all out of sync for
	336	* some periods in here!
	337	*/
e6b8fd02	338	_GLOBAL(__tm_recheckpoint)
98ae22e1 MN	339	mfcr r5
98ae22e1 MN	340	mflr r0
bbe30b3b	341	stw r5, 8(r1)
98ae22e1	342	std r0, 16(r1)
6403105b	343	std r2, STK_GOT(r1)
98ae22e1 MN	344	stdu r1, -TM_FRAME_SIZE(r1)
	345
	346	/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD].
	347	* This is used for backing up the NVGPRs:
	348	*/
	349	SAVE_NVGPRS(r1)
	350
98ae22e1 MN	351	/* Load complete register state from ts_ckpt* registers */
	352
	353	addi r7, r3, PT_CKPT_REGS /* Thread's ckpt_regs */
	354
	355	/* Make r7 look like an exception frame so that we
	356	* can use the neat GPRx(n) macros. r7 is now NOT a pt_regs ptr!
	357	*/
	358	subi r7, r7, STACK_FRAME_OVERHEAD
	359
eb5c3f1c	360	/* We need to setup MSR for FP/VMX/VSX register save instructions. */
98ae22e1	361	mfmsr r6
eb5c3f1c	362	mr r5, r6
98ae22e1	363	ori r5, r5, MSR_FP
eb5c3f1c CB	364	#ifdef CONFIG_ALTIVEC
	365	oris r5, r5, MSR_VEC@h
	366	#endif
98ae22e1 MN	367	#ifdef CONFIG_VSX
98ae22e1 MN	368	BEGIN_FTR_SECTION
eb5c3f1c	369	oris r5,r5, MSR_VSX@h
98ae22e1 MN	370	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
98ae22e1 MN	371	#endif
eb5c3f1c	372	mtmsrd r5
98ae22e1	373
f110c0c1	374	#ifdef CONFIG_ALTIVEC
dc310669 CB	375	/*
	376	* FP and VEC registers: These are recheckpointed from
	377	* thread.ckfp_state and thread.ckvr_state respectively. The
	378	* thread.fp_state[] version holds the 'live' (transactional)
	379	* and will be loaded subsequently by any FPUnavailable trap.
98ae22e1	380	*/
000ec280	381	addi r8, r3, THREAD_CKVRSTATE
de79f7b9	382	li r5, VRSTATE_VSCR
c2ce6f9f AB	383	lvx v0, r8, r5
c2ce6f9f AB	384	mtvscr v0
de79f7b9	385	REST_32VRS(0, r5, r8) /* r5 scratch, r8 ptr */
000ec280	386	ld r5, THREAD_CKVRSAVE(r3)
98ae22e1	387	mtspr SPRN_VRSAVE, r5
f110c0c1	388	#endif
98ae22e1	389
000ec280	390	addi r8, r3, THREAD_CKFPSTATE
de79f7b9	391	lfd fr0, FPSTATE_FPSCR(r8)
98ae22e1	392	MTFSF_L(fr0)
de79f7b9	393	REST_32FPRS_VSRS(0, R4, R8)
98ae22e1	394
98ae22e1 MN	395	mtmsr r6 /* FP/Vec off again! */
	396
	397	restore_gprs:
28e61cc4	398
e9bdc3d6 MN	399	/* ****************** CR,LR,CCR,MSR ******** */
	400	ld r4, _CTR(r7)
	401	ld r5, _LINK(r7)
e9bdc3d6	402	ld r8, _XER(r7)
28e61cc4	403
e9bdc3d6 MN	404	mtctr r4
e9bdc3d6 MN	405	mtlr r5
e9bdc3d6	406	mtxer r8
28e61cc4	407
e9bdc3d6 MN	408	/* ****************** TAR ****************** */
	409	ld r4, THREAD_TM_TAR(r3)
	410	mtspr SPRN_TAR, r4
98ae22e1	411
e9bdc3d6 MN	412	/* Load up the PPR and DSCR in GPRs only at this stage */
	413	ld r5, THREAD_TM_DSCR(r3)
	414	ld r6, THREAD_TM_PPR(r3)
98ae22e1	415
7f06f21d MN	416	REST_GPR(0, r7) /* GPR0 */
7f06f21d MN	417	REST_2GPRS(2, r7) /* GPR2-3 */
e9bdc3d6	418	REST_GPR(4, r7) /* GPR4 */
98ae22e1 MN	419	REST_4GPRS(8, r7) /* GPR8-11 */
	420	REST_2GPRS(12, r7) /* GPR12-13 */
	421
	422	REST_NVGPRS(r7) /* GPR14-31 */
	423
e9bdc3d6 MN	424	/* Load up PPR and DSCR here so we don't run with user values for long
	425	*/
	426	mtspr SPRN_DSCR, r5
	427	mtspr SPRN_PPR, r6
	428
7f06f21d MN	429	/* Do final sanity check on TEXASR to make sure FS is set. Do this
	430	* here before we load up the userspace r1 so any bugs we hit will get
	431	* a call chain */
	432	mfspr r5, SPRN_TEXASR
	433	srdi r5, r5, 16
	434	li r6, (TEXASR_FS)@h
	435	and r6, r6, r5
	436	1: tdeqi r6, 0
	437	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
	438
	439	/* Do final sanity check on MSR to make sure we are not transactional
	440	* or suspended
	441	*/
	442	mfmsr r6
	443	li r5, (MSR_TS_MASK)@higher
	444	srdi r6, r6, 32
	445	and r6, r6, r5
	446	1: tdnei r6, 0
	447	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
	448
	449	/* Restore CR */
	450	ld r6, _CCR(r7)
	451	mtcr r6
	452
e9bdc3d6	453	REST_GPR(6, r7)
190ce869 MN	454
	455	/*
	456	* Store r1 and r5 on the stack so that we can access them
	457	* after we clear MSR RI.
	458	*/
	459
	460	REST_GPR(5, r7)
	461	std r5, -8(r1)
	462	ld r5, GPR1(r7)
	463	std r5, -16(r1)
	464
	465	REST_GPR(7, r7)
	466
	467	/* Clear MSR RI since we are about to change r1. EE is already off */
	468	li r5, 0
	469	mtmsrd r5, 1
	470
	471	/*
	472	* BE CAREFUL HERE:
	473	* At this point we can't take an SLB miss since we have MSR_RI
	474	* off. Load only to/from the stack/paca which are in SLB bolted regions
	475	* until we turn MSR RI back on.
	476	*/
	477
6bcb8014	478	SET_SCRATCH0(r1)
190ce869 MN	479	ld r5, -8(r1)
190ce869 MN	480	ld r1, -16(r1)
98ae22e1 MN	481
	482	/* Commit register state as checkpointed state: */
	483	TRECHKPT
	484
e9bdc3d6 MN	485	HMT_MEDIUM
e9bdc3d6 MN	486
98ae22e1 MN	487	/* Our transactional state has now changed.
	488	*
	489	* Now just get out of here. Transactional (current) state will be
	490	* updated once restore is called on the return path in the _switch-ed
	491	* -to process.
	492	*/
	493
	494	GET_PACA(r13)
	495	GET_SCRATCH0(r1)
	496
090b9284 MN	497	/* R1 is restored, so we are recoverable again. EE is still off */
	498	li r4, MSR_RI
	499	mtmsrd r4, 1
	500
98ae22e1 MN	501	REST_NVGPRS(r1)
	502
	503	addi r1, r1, TM_FRAME_SIZE
bbe30b3b	504	lwz r4, 8(r1)
98ae22e1 MN	505	ld r0, 16(r1)
	506	mtcr r4
	507	mtlr r0
6403105b	508	ld r2, STK_GOT(r1)
e9bdc3d6	509
1739ea9e	510	/* Load CPU's default DSCR */
1db36525	511	ld r0, PACA_DSCR_DEFAULT(r13)
e9bdc3d6 MN	512	mtspr SPRN_DSCR, r0
e9bdc3d6 MN	513
98ae22e1 MN	514	blr
	515
	516	/* ****************************************************************** */