[linux-2.6-block.git] / arch / ia64 / kernel / minstate.h


#include <asm/cache.h>

#include "entry.h"

#ifdef CONFIG_VIRT_CPU_ACCOUNTING
/* read ar.itc in advance, and use it before leaving bank 0 */
#define ACCOUNT_GET_STAMP				\
(pUStk) mov.m r20=ar.itc;
#define ACCOUNT_SYS_ENTER				\
(pUStk) br.call.spnt rp=account_sys_enter		\
	;;
#else
#define ACCOUNT_GET_STAMP
#define ACCOUNT_SYS_ENTER
#endif

.section ".data.patch.rse", "a"
.previous

/*
 * DO_SAVE_MIN switches to the kernel stacks (if necessary) and saves
 * the minimum state necessary that allows us to turn psr.ic back
 * on.
 *
 * Assumed state upon entry:
 *	psr.ic: off
 *	r31:	contains saved predicates (pr)
 *
 * Upon exit, the state is as follows:
 *	psr.ic: off
 *	 r2 = points to &pt_regs.r16
 *	 r8 = contents of ar.ccv
 *	 r9 = contents of ar.csd
 *	r10 = contents of ar.ssd
 *	r11 = FPSR_DEFAULT
 *	r12 = kernel sp (kernel virtual address)
 *	r13 = points to current task_struct (kernel virtual address)
 *	p15 = TRUE if psr.i is set in cr.ipsr
 *	predicate registers (other than p2, p3, and p15), b6, r3, r14, r15:
 *		preserved
 *
 * Note that psr.ic is NOT turned on by this macro.  This is so that
 * we can pass interruption state as arguments to a handler.
 */
#define DO_SAVE_MIN(COVER,SAVE_IFS,EXTRA,WORKAROUND)						\
	mov r16=IA64_KR(CURRENT);	/* M */							\
	mov r27=ar.rsc;			/* M */							\
	mov r20=r1;			/* A */							\
	mov r25=ar.unat;		/* M */							\
	mov r29=cr.ipsr;		/* M */							\
	mov r26=ar.pfs;			/* I */							\
	mov r28=cr.iip;			/* M */							\
	mov r21=ar.fpsr;		/* M */							\
	COVER;				/* B;; (or nothing) */					\
	;;											\
	adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16;						\
	;;											\
	ld1 r17=[r16];				/* load current->thread.on_ustack flag */	\
	st1 [r16]=r0;				/* clear current->thread.on_ustack flag */	\
	adds r1=-IA64_TASK_THREAD_ON_USTACK_OFFSET,r16						\
	/* switch from user to kernel RBS: */							\
	;;											\
	invala;				/* M */							\
	SAVE_IFS;										\
	cmp.eq pKStk,pUStk=r0,r17;		/* are we in kernel mode already? */		\
	;;											\
(pUStk)	mov ar.rsc=0;		/* set enforced lazy mode, pl 0, little-endian, loadrs=0 */	\
	;;											\
(pUStk)	mov.m r24=ar.rnat;									\
(pUStk)	addl r22=IA64_RBS_OFFSET,r1;			/* compute base of RBS */		\
(pKStk) mov r1=sp;					/* get sp  */				\
	;;											\
(pUStk) lfetch.fault.excl.nt1 [r22];								\
(pUStk)	addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1;	/* compute base of memory stack */	\
(pUStk)	mov r23=ar.bspstore;				/* save ar.bspstore */			\
	;;											\
(pUStk)	mov ar.bspstore=r22;				/* switch to kernel RBS */		\
(pKStk) addl r1=-IA64_PT_REGS_SIZE,r1;			/* if in kernel mode, use sp (r12) */	\
	;;											\
(pUStk)	mov r18=ar.bsp;										\
(pUStk)	mov ar.rsc=0x3;		/* set eager mode, pl 0, little-endian, loadrs=0 */		\
	adds r17=2*L1_CACHE_BYTES,r1;		/* really: biggest cache-line size */		\
	adds r16=PT(CR_IPSR),r1;								\
	;;											\
	lfetch.fault.excl.nt1 [r17],L1_CACHE_BYTES;						\
	st8 [r16]=r29;		/* save cr.ipsr */						\
	;;											\
	lfetch.fault.excl.nt1 [r17];								\
	tbit.nz p15,p0=r29,IA64_PSR_I_BIT;							\
	mov r29=b0										\
	;;											\
	WORKAROUND;										\
	adds r16=PT(R8),r1;	/* initialize first base pointer */				\
	adds r17=PT(R9),r1;	/* initialize second base pointer */				\
(pKStk)	mov r18=r0;		/* make sure r18 isn't NaT */					\
	;;											\
.mem.offset 0,0; st8.spill [r16]=r8,16;								\
.mem.offset 8,0; st8.spill [r17]=r9,16;								\
        ;;											\
.mem.offset 0,0; st8.spill [r16]=r10,24;							\
.mem.offset 8,0; st8.spill [r17]=r11,24;							\
        ;;											\
	st8 [r16]=r28,16;	/* save cr.iip */						\
	st8 [r17]=r30,16;	/* save cr.ifs */						\
(pUStk)	sub r18=r18,r22;	/* r18=RSE.ndirty*8 */						\
	mov r8=ar.ccv;										\
	mov r9=ar.csd;										\
	mov r10=ar.ssd;										\
	movl r11=FPSR_DEFAULT;   /* L-unit */							\
	;;											\
	st8 [r16]=r25,16;	/* save ar.unat */						\
	st8 [r17]=r26,16;	/* save ar.pfs */						\
	shl r18=r18,16;		/* compute ar.rsc to be used for "loadrs" */			\
	;;											\
	st8 [r16]=r27,16;	/* save ar.rsc */						\
(pUStk)	st8 [r17]=r24,16;	/* save ar.rnat */						\
(pKStk)	adds r17=16,r17;	/* skip over ar_rnat field */					\
	;;			/* avoid RAW on r16 & r17 */					\
(pUStk)	st8 [r16]=r23,16;	/* save ar.bspstore */						\
	st8 [r17]=r31,16;	/* save predicates */						\
(pKStk)	adds r16=16,r16;	/* skip over ar_bspstore field */				\
	;;											\
	st8 [r16]=r29,16;	/* save b0 */							\
	st8 [r17]=r18,16;	/* save ar.rsc value for "loadrs" */				\
	cmp.eq pNonSys,pSys=r0,r0	/* initialize pSys=0, pNonSys=1 */			\
	;;											\
.mem.offset 0,0; st8.spill [r16]=r20,16;	/* save original r1 */				\
.mem.offset 8,0; st8.spill [r17]=r12,16;							\
	adds r12=-16,r1;	/* switch to kernel memory stack (with 16 bytes of scratch) */	\
	;;											\
.mem.offset 0,0; st8.spill [r16]=r13,16;							\
.mem.offset 8,0; st8.spill [r17]=r21,16;	/* save ar.fpsr */				\
	mov r13=IA64_KR(CURRENT);	/* establish `current' */				\
	;;											\
.mem.offset 0,0; st8.spill [r16]=r15,16;							\
.mem.offset 8,0; st8.spill [r17]=r14,16;							\
	;;											\
.mem.offset 0,0; st8.spill [r16]=r2,16;								\
.mem.offset 8,0; st8.spill [r17]=r3,16;								\
	ACCOUNT_GET_STAMP									\
	adds r2=IA64_PT_REGS_R16_OFFSET,r1;							\
	;;											\
	EXTRA;											\
	movl r1=__gp;		/* establish kernel global pointer */				\
	;;											\
	ACCOUNT_SYS_ENTER									\
	bsw.1;			/* switch back to bank 1 (must be last in insn group) */	\
	;;

/*
 * SAVE_REST saves the remainder of pt_regs (with psr.ic on).
 *
 * Assumed state upon entry:
 *	psr.ic: on
 *	r2:	points to &pt_regs.r16
 *	r3:	points to &pt_regs.r17
 *	r8:	contents of ar.ccv
 *	r9:	contents of ar.csd
 *	r10:	contents of ar.ssd
 *	r11:	FPSR_DEFAULT
 *
 * Registers r14 and r15 are guaranteed not to be touched by SAVE_REST.
 */
#define SAVE_REST				\
.mem.offset 0,0; st8.spill [r2]=r16,16;		\
.mem.offset 8,0; st8.spill [r3]=r17,16;		\
	;;					\
.mem.offset 0,0; st8.spill [r2]=r18,16;		\
.mem.offset 8,0; st8.spill [r3]=r19,16;		\
	;;					\
.mem.offset 0,0; st8.spill [r2]=r20,16;		\
.mem.offset 8,0; st8.spill [r3]=r21,16;		\
	mov r18=b6;				\
	;;					\
.mem.offset 0,0; st8.spill [r2]=r22,16;		\
.mem.offset 8,0; st8.spill [r3]=r23,16;		\
	mov r19=b7;				\
	;;					\
.mem.offset 0,0; st8.spill [r2]=r24,16;		\
.mem.offset 8,0; st8.spill [r3]=r25,16;		\
	;;					\
.mem.offset 0,0; st8.spill [r2]=r26,16;		\
.mem.offset 8,0; st8.spill [r3]=r27,16;		\
	;;					\
.mem.offset 0,0; st8.spill [r2]=r28,16;		\
.mem.offset 8,0; st8.spill [r3]=r29,16;		\
	;;					\
.mem.offset 0,0; st8.spill [r2]=r30,16;		\
.mem.offset 8,0; st8.spill [r3]=r31,32;		\
	;;					\
	mov ar.fpsr=r11;	/* M-unit */	\
	st8 [r2]=r8,8;		/* ar.ccv */	\
	adds r24=PT(B6)-PT(F7),r3;		\
	;;					\
	stf.spill [r2]=f6,32;			\
	stf.spill [r3]=f7,32;			\
	;;					\
	stf.spill [r2]=f8,32;			\
	stf.spill [r3]=f9,32;			\
	;;					\
	stf.spill [r2]=f10;			\
	stf.spill [r3]=f11;			\
	adds r25=PT(B7)-PT(F11),r3;		\
	;;					\
	st8 [r24]=r18,16;       /* b6 */	\
	st8 [r25]=r19,16;       /* b7 */	\
	;;					\
	st8 [r24]=r9;        	/* ar.csd */	\
	st8 [r25]=r10;      	/* ar.ssd */	\
	;;

#define RSE_WORKAROUND				\
(pUStk) extr.u r17=r18,3,6;			\
(pUStk)	sub r16=r18,r22;			\
[1:](pKStk)	br.cond.sptk.many 1f;		\
	.xdata4 ".data.patch.rse",1b-.		\
	;;					\
	cmp.ge p6,p7 = 33,r17;			\
	;;					\
(p6)	mov r17=0x310;				\
(p7)	mov r17=0x308;				\
	;;					\
	cmp.leu p1,p0=r16,r17;			\
(p1)	br.cond.sptk.many 1f;			\
	dep.z r17=r26,0,62;			\
	movl r16=2f;				\
	;;					\
	mov ar.pfs=r17;				\
	dep r27=r0,r27,16,14;			\
	mov b0=r16;				\
	;;					\
	br.ret.sptk b0;				\
	;;					\
2:						\
	mov ar.rsc=r0				\
	;;					\
	flushrs;				\
	;;					\
	mov ar.bspstore=r22			\
	;;					\
	mov r18=ar.bsp;				\
	;;					\
1:						\
	.pred.rel "mutex", pKStk, pUStk

#define SAVE_MIN_WITH_COVER	DO_SAVE_MIN(cover, mov r30=cr.ifs, , RSE_WORKAROUND)
#define SAVE_MIN_WITH_COVER_R19	DO_SAVE_MIN(cover, mov r30=cr.ifs, mov r15=r19, RSE_WORKAROUND)
#define SAVE_MIN			DO_SAVE_MIN(     , mov r30=r0, , )
Commit	Line	Data
1da177e4 LT	1
	2	#include <asm/cache.h>
	3
	4	#include "entry.h"
	5
b64f34cd HS	6	#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	7	/* read ar.itc in advance, and use it before leaving bank 0 */
	8	#define ACCOUNT_GET_STAMP \
	9	(pUStk) mov.m r20=ar.itc;
	10	#define ACCOUNT_SYS_ENTER \
	11	(pUStk) br.call.spnt rp=account_sys_enter \
	12	;;
	13	#else
	14	#define ACCOUNT_GET_STAMP
	15	#define ACCOUNT_SYS_ENTER
	16	#endif
	17
4dcc29e1 TL	18	.section ".data.patch.rse", "a"
	19	.previous
	20
1da177e4 LT	21	/*
	22	* DO_SAVE_MIN switches to the kernel stacks (if necessary) and saves
	23	* the minimum state necessary that allows us to turn psr.ic back
	24	* on.
	25	*
	26	* Assumed state upon entry:
	27	* psr.ic: off
	28	* r31: contains saved predicates (pr)
	29	*
	30	* Upon exit, the state is as follows:
	31	* psr.ic: off
	32	* r2 = points to &pt_regs.r16
	33	* r8 = contents of ar.ccv
	34	* r9 = contents of ar.csd
	35	* r10 = contents of ar.ssd
	36	* r11 = FPSR_DEFAULT
	37	* r12 = kernel sp (kernel virtual address)
	38	* r13 = points to current task_struct (kernel virtual address)
	39	* p15 = TRUE if psr.i is set in cr.ipsr
	40	* predicate registers (other than p2, p3, and p15), b6, r3, r14, r15:
	41	* preserved
	42	*
	43	* Note that psr.ic is NOT turned on by this macro. This is so that
	44	* we can pass interruption state as arguments to a handler.
	45	*/
4dcc29e1	46	#define DO_SAVE_MIN(COVER,SAVE_IFS,EXTRA,WORKAROUND) \
05f335ea	47	mov r16=IA64_KR(CURRENT); /* M */ \
1da177e4 LT	48	mov r27=ar.rsc; /* M */ \
	49	mov r20=r1; /* A */ \
	50	mov r25=ar.unat; /* M */ \
	51	mov r29=cr.ipsr; /* M */ \
	52	mov r26=ar.pfs; /* I */ \
	53	mov r28=cr.iip; /* M */ \
	54	mov r21=ar.fpsr; /* M */ \
	55	COVER; /* B;; (or nothing) */ \
	56	;; \
	57	adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16; \
	58	;; \
	59	ld1 r17=[r16]; /* load current->thread.on_ustack flag */ \
	60	st1 [r16]=r0; /* clear current->thread.on_ustack flag */ \
	61	adds r1=-IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 \
	62	/* switch from user to kernel RBS: */ \
	63	;; \
	64	invala; /* M */ \
	65	SAVE_IFS; \
	66	cmp.eq pKStk,pUStk=r0,r17; /* are we in kernel mode already? */ \
	67	;; \
05f335ea KO	68	(pUStk) mov ar.rsc=0; /* set enforced lazy mode, pl 0, little-endian, loadrs=0 */ \
	69	;; \
	70	(pUStk) mov.m r24=ar.rnat; \
	71	(pUStk) addl r22=IA64_RBS_OFFSET,r1; /* compute base of RBS */ \
	72	(pKStk) mov r1=sp; /* get sp */ \
	73	;; \
	74	(pUStk) lfetch.fault.excl.nt1 [r22]; \
	75	(pUStk) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1; /* compute base of memory stack */ \
	76	(pUStk) mov r23=ar.bspstore; /* save ar.bspstore */ \
	77	;; \
	78	(pUStk) mov ar.bspstore=r22; /* switch to kernel RBS */ \
	79	(pKStk) addl r1=-IA64_PT_REGS_SIZE,r1; /* if in kernel mode, use sp (r12) */ \
	80	;; \
	81	(pUStk) mov r18=ar.bsp; \
	82	(pUStk) mov ar.rsc=0x3; /* set eager mode, pl 0, little-endian, loadrs=0 */ \
1da177e4 LT	83	adds r17=2L1_CACHE_BYTES,r1; / really: biggest cache-line size */ \
	84	adds r16=PT(CR_IPSR),r1; \
	85	;; \
	86	lfetch.fault.excl.nt1 [r17],L1_CACHE_BYTES; \
	87	st8 [r16]=r29; /* save cr.ipsr */ \
	88	;; \
	89	lfetch.fault.excl.nt1 [r17]; \
	90	tbit.nz p15,p0=r29,IA64_PSR_I_BIT; \
	91	mov r29=b0 \
	92	;; \
4dcc29e1	93	WORKAROUND; \
1da177e4 LT	94	adds r16=PT(R8),r1; /* initialize first base pointer */ \
	95	adds r17=PT(R9),r1; /* initialize second base pointer */ \
	96	(pKStk) mov r18=r0; /* make sure r18 isn't NaT */ \
	97	;; \
	98	.mem.offset 0,0; st8.spill [r16]=r8,16; \
	99	.mem.offset 8,0; st8.spill [r17]=r9,16; \
	100	;; \
	101	.mem.offset 0,0; st8.spill [r16]=r10,24; \
	102	.mem.offset 8,0; st8.spill [r17]=r11,24; \
	103	;; \
	104	st8 [r16]=r28,16; /* save cr.iip */ \
	105	st8 [r17]=r30,16; /* save cr.ifs */ \
	106	(pUStk) sub r18=r18,r22; /* r18=RSE.ndirty8 / \
	107	mov r8=ar.ccv; \
	108	mov r9=ar.csd; \
	109	mov r10=ar.ssd; \
	110	movl r11=FPSR_DEFAULT; /* L-unit */ \
	111	;; \
	112	st8 [r16]=r25,16; /* save ar.unat */ \
	113	st8 [r17]=r26,16; /* save ar.pfs */ \
	114	shl r18=r18,16; /* compute ar.rsc to be used for "loadrs" */ \
	115	;; \
	116	st8 [r16]=r27,16; /* save ar.rsc */ \
	117	(pUStk) st8 [r17]=r24,16; /* save ar.rnat */ \
	118	(pKStk) adds r17=16,r17; /* skip over ar_rnat field */ \
	119	;; /* avoid RAW on r16 & r17 */ \
	120	(pUStk) st8 [r16]=r23,16; /* save ar.bspstore */ \
	121	st8 [r17]=r31,16; /* save predicates */ \
	122	(pKStk) adds r16=16,r16; /* skip over ar_bspstore field */ \
	123	;; \
	124	st8 [r16]=r29,16; /* save b0 */ \
	125	st8 [r17]=r18,16; /* save ar.rsc value for "loadrs" */ \
	126	cmp.eq pNonSys,pSys=r0,r0 /* initialize pSys=0, pNonSys=1 */ \
	127	;; \
	128	.mem.offset 0,0; st8.spill [r16]=r20,16; /* save original r1 */ \
	129	.mem.offset 8,0; st8.spill [r17]=r12,16; \
	130	adds r12=-16,r1; /* switch to kernel memory stack (with 16 bytes of scratch) */ \
	131	;; \
	132	.mem.offset 0,0; st8.spill [r16]=r13,16; \
	133	.mem.offset 8,0; st8.spill [r17]=r21,16; /* save ar.fpsr */ \
	134	mov r13=IA64_KR(CURRENT); /* establish `current' */ \
	135	;; \
	136	.mem.offset 0,0; st8.spill [r16]=r15,16; \
	137	.mem.offset 8,0; st8.spill [r17]=r14,16; \
	138	;; \
	139	.mem.offset 0,0; st8.spill [r16]=r2,16; \
	140	.mem.offset 8,0; st8.spill [r17]=r3,16; \
b64f34cd	141	ACCOUNT_GET_STAMP \
1da177e4 LT	142	adds r2=IA64_PT_REGS_R16_OFFSET,r1; \
	143	;; \
	144	EXTRA; \
	145	movl r1=__gp; /* establish kernel global pointer */ \
	146	;; \
b64f34cd	147	ACCOUNT_SYS_ENTER \
05f335ea KO	148	bsw.1; /* switch back to bank 1 (must be last in insn group) */ \
05f335ea KO	149	;;
1da177e4 LT	150
	151	/*
	152	* SAVE_REST saves the remainder of pt_regs (with psr.ic on).
	153	*
	154	* Assumed state upon entry:
	155	* psr.ic: on
	156	* r2: points to &pt_regs.r16
	157	* r3: points to &pt_regs.r17
	158	* r8: contents of ar.ccv
	159	* r9: contents of ar.csd
	160	* r10: contents of ar.ssd
	161	* r11: FPSR_DEFAULT
	162	*
	163	* Registers r14 and r15 are guaranteed not to be touched by SAVE_REST.
	164	*/
	165	#define SAVE_REST \
	166	.mem.offset 0,0; st8.spill [r2]=r16,16; \
	167	.mem.offset 8,0; st8.spill [r3]=r17,16; \
	168	;; \
	169	.mem.offset 0,0; st8.spill [r2]=r18,16; \
	170	.mem.offset 8,0; st8.spill [r3]=r19,16; \
	171	;; \
	172	.mem.offset 0,0; st8.spill [r2]=r20,16; \
	173	.mem.offset 8,0; st8.spill [r3]=r21,16; \
	174	mov r18=b6; \
	175	;; \
	176	.mem.offset 0,0; st8.spill [r2]=r22,16; \
	177	.mem.offset 8,0; st8.spill [r3]=r23,16; \
	178	mov r19=b7; \
	179	;; \
	180	.mem.offset 0,0; st8.spill [r2]=r24,16; \
	181	.mem.offset 8,0; st8.spill [r3]=r25,16; \
	182	;; \
	183	.mem.offset 0,0; st8.spill [r2]=r26,16; \
	184	.mem.offset 8,0; st8.spill [r3]=r27,16; \
	185	;; \
	186	.mem.offset 0,0; st8.spill [r2]=r28,16; \
	187	.mem.offset 8,0; st8.spill [r3]=r29,16; \
	188	;; \
	189	.mem.offset 0,0; st8.spill [r2]=r30,16; \
	190	.mem.offset 8,0; st8.spill [r3]=r31,32; \
	191	;; \
	192	mov ar.fpsr=r11; /* M-unit */ \
	193	st8 [r2]=r8,8; /* ar.ccv */ \
	194	adds r24=PT(B6)-PT(F7),r3; \
	195	;; \
	196	stf.spill [r2]=f6,32; \
	197	stf.spill [r3]=f7,32; \
	198	;; \
	199	stf.spill [r2]=f8,32; \
	200	stf.spill [r3]=f9,32; \
	201	;; \
	202	stf.spill [r2]=f10; \
	203	stf.spill [r3]=f11; \
	204	adds r25=PT(B7)-PT(F11),r3; \
	205	;; \
	206	st8 [r24]=r18,16; /* b6 */ \
	207	st8 [r25]=r19,16; /* b7 */ \
	208	;; \
	209	st8 [r24]=r9; /* ar.csd */ \
	210	st8 [r25]=r10; /* ar.ssd */ \
	211	;;
	212
4dcc29e1 TL	213	#define RSE_WORKAROUND \
	214	(pUStk) extr.u r17=r18,3,6; \
	215	(pUStk) sub r16=r18,r22; \
	216	[1:](pKStk) br.cond.sptk.many 1f; \
	217	.xdata4 ".data.patch.rse",1b-. \
	218	;; \
	219	cmp.ge p6,p7 = 33,r17; \
	220	;; \
	221	(p6) mov r17=0x310; \
	222	(p7) mov r17=0x308; \
	223	;; \
	224	cmp.leu p1,p0=r16,r17; \
	225	(p1) br.cond.sptk.many 1f; \
	226	dep.z r17=r26,0,62; \
	227	movl r16=2f; \
	228	;; \
	229	mov ar.pfs=r17; \
	230	dep r27=r0,r27,16,14; \
	231	mov b0=r16; \
	232	;; \
	233	br.ret.sptk b0; \
	234	;; \
	235	2: \
	236	mov ar.rsc=r0 \
	237	;; \
	238	flushrs; \
	239	;; \
	240	mov ar.bspstore=r22 \
	241	;; \
	242	mov r18=ar.bsp; \
	243	;; \
	244	1: \
	245	.pred.rel "mutex", pKStk, pUStk
	246
	247	#define SAVE_MIN_WITH_COVER DO_SAVE_MIN(cover, mov r30=cr.ifs, , RSE_WORKAROUND)
	248	#define SAVE_MIN_WITH_COVER_R19 DO_SAVE_MIN(cover, mov r30=cr.ifs, mov r15=r19, RSE_WORKAROUND)
	249	#define SAVE_MIN DO_SAVE_MIN( , mov r30=r0, , )