[linux-2.6-block.git] / arch / powerpc / include / asm / exception-64e.h

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *  Definitions for use by exception code on Book3-E
 *
 *  Copyright (C) 2008 Ben. Herrenschmidt (benh@kernel.crashing.org), IBM Corp.
 */
#ifndef _ASM_POWERPC_EXCEPTION_64E_H
#define _ASM_POWERPC_EXCEPTION_64E_H

/*
 * SPRGs usage an other considerations...
 *
 * Since TLB miss and other standard exceptions can be interrupted by
 * critical exceptions which can themselves be interrupted by machine
 * checks, and since the two later can themselves cause a TLB miss when
 * hitting the linear mapping for the kernel stacks, we need to be a bit
 * creative on how we use SPRGs.
 *
 * The base idea is that we have one SRPG reserved for critical and one
 * for machine check interrupts. Those are used to save a GPR that can
 * then be used to get the PACA, and store as much context as we need
 * to save in there. That includes saving the SPRGs used by the TLB miss
 * handler for linear mapping misses and the associated SRR0/1 due to
 * the above re-entrancy issue.
 *
 * So here's the current usage pattern. It's done regardless of which
 * SPRGs are user-readable though, thus we might have to change some of
 * this later. In order to do that more easily, we use special constants
 * for naming them
 *
 * WARNING: Some of these SPRGs are user readable. We need to do something
 * about it as some point by making sure they can't be used to leak kernel
 * critical data
 */

#define PACA_EXGDBELL PACA_EXGEN

/* We are out of SPRGs so we save some things in the PACA. The normal
 * exception frame is smaller than the CRIT or MC one though
 */
#define EX_R1		(0 * 8)
#define EX_CR		(1 * 8)
#define EX_R10		(2 * 8)
#define EX_R11		(3 * 8)
#define EX_R14		(4 * 8)
#define EX_R15		(5 * 8)

/*
 * The TLB miss exception uses different slots.
 *
 * The bolted variant uses only the first six fields,
 * which in combination with pgd and kernel_pgd fits in
 * one 64-byte cache line.
 */

#define EX_TLB_R10	( 0 * 8)
#define EX_TLB_R11	( 1 * 8)
#define EX_TLB_R14	( 2 * 8)
#define EX_TLB_R15	( 3 * 8)
#define EX_TLB_R16	( 4 * 8)
#define EX_TLB_CR	( 5 * 8)
#define EX_TLB_R12	( 6 * 8)
#define EX_TLB_R13	( 7 * 8)
#define EX_TLB_DEAR	( 8 * 8) /* Level 0 and 2 only */
#define EX_TLB_ESR	( 9 * 8) /* Level 0 and 2 only */
#define EX_TLB_SRR0	(10 * 8)
#define EX_TLB_SRR1	(11 * 8)
#define EX_TLB_R7	(12 * 8)
#ifdef CONFIG_BOOK3E_MMU_TLB_STATS
#define EX_TLB_R8	(13 * 8)
#define EX_TLB_R9	(14 * 8)
#define EX_TLB_LR	(15 * 8)
#define EX_TLB_SIZE	(16 * 8)
#else
#define EX_TLB_SIZE	(13 * 8)
#endif

#define	START_EXCEPTION(label)						\
	.globl exc_##label##_book3e;					\
exc_##label##_book3e:

/* TLB miss exception prolog
 *
 * This prolog handles re-entrancy (up to 3 levels supported in the PACA
 * though we currently don't test for overflow). It provides you with a
 * re-entrancy safe working space of r10...r16 and CR with r12 being used
 * as the exception area pointer in the PACA for that level of re-entrancy
 * and r13 containing the PACA pointer.
 *
 * SRR0 and SRR1 are saved, but DEAR and ESR are not, since they don't apply
 * as-is for instruction exceptions. It's up to the actual exception code
 * to save them as well if required.
 */
#define TLB_MISS_PROLOG							    \
	mtspr	SPRN_SPRG_TLB_SCRATCH,r12;				    \
	mfspr	r12,SPRN_SPRG_TLB_EXFRAME;				    \
	std	r10,EX_TLB_R10(r12);					    \
	mfcr	r10;							    \
	std	r11,EX_TLB_R11(r12);					    \
	mfspr	r11,SPRN_SPRG_TLB_SCRATCH;				    \
	std	r13,EX_TLB_R13(r12);					    \
	mfspr	r13,SPRN_SPRG_PACA;					    \
	std	r14,EX_TLB_R14(r12);					    \
	addi	r14,r12,EX_TLB_SIZE;					    \
	std	r15,EX_TLB_R15(r12);					    \
	mfspr	r15,SPRN_SRR1;						    \
	std	r16,EX_TLB_R16(r12);					    \
	mfspr	r16,SPRN_SRR0;						    \
	std	r10,EX_TLB_CR(r12);					    \
	std	r11,EX_TLB_R12(r12);					    \
	mtspr	SPRN_SPRG_TLB_EXFRAME,r14;				    \
	std	r15,EX_TLB_SRR1(r12);					    \
	std	r16,EX_TLB_SRR0(r12);					    \
	TLB_MISS_PROLOG_STATS

/* And these are the matching epilogs that restores things
 *
 * There are 3 epilogs:
 *
 * - SUCCESS       : Unwinds one level
 * - ERROR         : restore from level 0 and reset
 * - ERROR_SPECIAL : restore from current level and reset
 *
 * Normal errors use ERROR, that is, they restore the initial fault context
 * and trigger a fault. However, there is a special case for linear mapping
 * errors. Those should basically never happen, but if they do happen, we
 * want the error to point out the context that did that linear mapping
 * fault, not the initial level 0 (basically, we got a bogus PGF or something
 * like that). For userland errors on the linear mapping, there is no
 * difference since those are always level 0 anyway
 */

#define TLB_MISS_RESTORE(freg)						    \
	ld	r14,EX_TLB_CR(r12);					    \
	ld	r10,EX_TLB_R10(r12);					    \
	ld	r15,EX_TLB_SRR0(r12);					    \
	ld	r16,EX_TLB_SRR1(r12);					    \
	mtspr	SPRN_SPRG_TLB_EXFRAME,freg;				    \
	ld	r11,EX_TLB_R11(r12);					    \
	mtcr	r14;							    \
	ld	r13,EX_TLB_R13(r12);					    \
	ld	r14,EX_TLB_R14(r12);					    \
	mtspr	SPRN_SRR0,r15;						    \
	ld	r15,EX_TLB_R15(r12);					    \
	mtspr	SPRN_SRR1,r16;						    \
	TLB_MISS_RESTORE_STATS						    \
	ld	r16,EX_TLB_R16(r12);					    \
	ld	r12,EX_TLB_R12(r12);					    \

#define TLB_MISS_EPILOG_SUCCESS						    \
	TLB_MISS_RESTORE(r12)

#define TLB_MISS_EPILOG_ERROR						    \
	addi	r12,r13,PACA_EXTLB;					    \
	TLB_MISS_RESTORE(r12)

#define TLB_MISS_EPILOG_ERROR_SPECIAL					    \
	addi	r11,r13,PACA_EXTLB;					    \
	TLB_MISS_RESTORE(r11)

#ifdef CONFIG_BOOK3E_MMU_TLB_STATS
#define TLB_MISS_PROLOG_STATS						    \
	mflr	r10;							    \
	std	r8,EX_TLB_R8(r12);					    \
	std	r9,EX_TLB_R9(r12);					    \
	std	r10,EX_TLB_LR(r12);
#define TLB_MISS_RESTORE_STATS					            \
	ld	r16,EX_TLB_LR(r12);					    \
	ld	r9,EX_TLB_R9(r12);					    \
	ld	r8,EX_TLB_R8(r12);					    \
	mtlr	r16;
#define TLB_MISS_STATS_D(name)						    \
	addi	r9,r13,MMSTAT_DSTATS+name;				    \
	bl	tlb_stat_inc;
#define TLB_MISS_STATS_I(name)						    \
	addi	r9,r13,MMSTAT_ISTATS+name;				    \
	bl	tlb_stat_inc;
#define TLB_MISS_STATS_X(name)						    \
	ld	r8,PACA_EXTLB+EX_TLB_ESR(r13);				    \
	cmpdi	cr2,r8,-1;						    \
	beq	cr2,61f;						    \
	addi	r9,r13,MMSTAT_DSTATS+name;				    \
	b	62f;							    \
61:	addi	r9,r13,MMSTAT_ISTATS+name;				    \
62:	bl	tlb_stat_inc;
#define TLB_MISS_STATS_SAVE_INFO					    \
	std	r14,EX_TLB_ESR(r12);	/* save ESR */
#define TLB_MISS_STATS_SAVE_INFO_BOLTED					    \
	std	r14,PACA_EXTLB+EX_TLB_ESR(r13);	/* save ESR */
#else
#define TLB_MISS_PROLOG_STATS
#define TLB_MISS_RESTORE_STATS
#define TLB_MISS_PROLOG_STATS_BOLTED
#define TLB_MISS_RESTORE_STATS_BOLTED
#define TLB_MISS_STATS_D(name)
#define TLB_MISS_STATS_I(name)
#define TLB_MISS_STATS_X(name)
#define TLB_MISS_STATS_Y(name)
#define TLB_MISS_STATS_SAVE_INFO
#define TLB_MISS_STATS_SAVE_INFO_BOLTED
#endif

#define SET_IVOR(vector_number, vector_offset)	\
	LOAD_REG_ADDR(r3,interrupt_base_book3e);\
	ori	r3,r3,vector_offset@l;		\
	mtspr	SPRN_IVOR##vector_number,r3;

#define RFI_TO_KERNEL							\
	rfi

#define RFI_TO_USER							\
	rfi

#endif /* _ASM_POWERPC_EXCEPTION_64E_H */
Commit	Line	Data
2874c5fd	1	/* SPDX-License-Identifier: GPL-2.0-or-later */
13363ab9 BH	2	/*
	3	* Definitions for use by exception code on Book3-E
	4	*
	5	* Copyright (C) 2008 Ben. Herrenschmidt (benh@kernel.crashing.org), IBM Corp.
13363ab9 BH	6	*/
	7	#ifndef _ASM_POWERPC_EXCEPTION_64E_H
	8	#define _ASM_POWERPC_EXCEPTION_64E_H
	9
	10	/*
	11	* SPRGs usage an other considerations...
	12	*
	13	* Since TLB miss and other standard exceptions can be interrupted by
	14	* critical exceptions which can themselves be interrupted by machine
	15	* checks, and since the two later can themselves cause a TLB miss when
	16	* hitting the linear mapping for the kernel stacks, we need to be a bit
	17	* creative on how we use SPRGs.
	18	*
	19	* The base idea is that we have one SRPG reserved for critical and one
	20	* for machine check interrupts. Those are used to save a GPR that can
	21	* then be used to get the PACA, and store as much context as we need
	22	* to save in there. That includes saving the SPRGs used by the TLB miss
	23	* handler for linear mapping misses and the associated SRR0/1 due to
	24	* the above re-entrancy issue.
	25	*
	26	* So here's the current usage pattern. It's done regardless of which
	27	* SPRGs are user-readable though, thus we might have to change some of
	28	* this later. In order to do that more easily, we use special constants
	29	* for naming them
	30	*
	31	* WARNING: Some of these SPRGs are user readable. We need to do something
	32	* about it as some point by making sure they can't be used to leak kernel
	33	* critical data
	34	*/
	35
5473eb1c	36	#define PACA_EXGDBELL PACA_EXGEN
13363ab9 BH	37
	38	/* We are out of SPRGs so we save some things in the PACA. The normal
	39	* exception frame is smaller than the CRIT or MC one though
	40	*/
	41	#define EX_R1 (0 * 8)
	42	#define EX_CR (1 * 8)
	43	#define EX_R10 (2 * 8)
	44	#define EX_R11 (3 * 8)
9d378dfa SW	45	#define EX_R14 (4 * 8)
9d378dfa SW	46	#define EX_R15 (5 * 8)
13363ab9	47
f67f4ef5 SW	48	/*
	49	* The TLB miss exception uses different slots.
	50	*
	51	* The bolted variant uses only the first six fields,
	52	* which in combination with pgd and kernel_pgd fits in
	53	* one 64-byte cache line.
	54	*/
13363ab9 BH	55
	56	#define EX_TLB_R10 ( 0 * 8)
	57	#define EX_TLB_R11 ( 1 * 8)
f67f4ef5 SW	58	#define EX_TLB_R14 ( 2 * 8)
	59	#define EX_TLB_R15 ( 3 * 8)
	60	#define EX_TLB_R16 ( 4 * 8)
	61	#define EX_TLB_CR ( 5 * 8)
	62	#define EX_TLB_R12 ( 6 * 8)
	63	#define EX_TLB_R13 ( 7 * 8)
13363ab9 BH	64	#define EX_TLB_DEAR ( 8 * 8) /* Level 0 and 2 only */
	65	#define EX_TLB_ESR ( 9 * 8) /* Level 0 and 2 only */
	66	#define EX_TLB_SRR0 (10 * 8)
	67	#define EX_TLB_SRR1 (11 * 8)
e1f580e8	68	#define EX_TLB_R7 (12 * 8)
13363ab9	69	#ifdef CONFIG_BOOK3E_MMU_TLB_STATS
e1f580e8 KH	70	#define EX_TLB_R8 (13 * 8)
	71	#define EX_TLB_R9 (14 * 8)
	72	#define EX_TLB_LR (15 * 8)
	73	#define EX_TLB_SIZE (16 * 8)
13363ab9	74	#else
e1f580e8	75	#define EX_TLB_SIZE (13 * 8)
13363ab9 BH	76	#endif
	77
	78	#define START_EXCEPTION(label) \
	79	.globl exc_##label##_book3e; \
	80	exc_##label##_book3e:
	81
	82	/* TLB miss exception prolog
	83	*
	84	* This prolog handles re-entrancy (up to 3 levels supported in the PACA
	85	* though we currently don't test for overflow). It provides you with a
	86	* re-entrancy safe working space of r10...r16 and CR with r12 being used
	87	* as the exception area pointer in the PACA for that level of re-entrancy
	88	* and r13 containing the PACA pointer.
	89	*
	90	* SRR0 and SRR1 are saved, but DEAR and ESR are not, since they don't apply
	91	* as-is for instruction exceptions. It's up to the actual exception code
	92	* to save them as well if required.
	93	*/
	94	#define TLB_MISS_PROLOG \
	95	mtspr SPRN_SPRG_TLB_SCRATCH,r12; \
	96	mfspr r12,SPRN_SPRG_TLB_EXFRAME; \
	97	std r10,EX_TLB_R10(r12); \
	98	mfcr r10; \
	99	std r11,EX_TLB_R11(r12); \
	100	mfspr r11,SPRN_SPRG_TLB_SCRATCH; \
	101	std r13,EX_TLB_R13(r12); \
	102	mfspr r13,SPRN_SPRG_PACA; \
	103	std r14,EX_TLB_R14(r12); \
	104	addi r14,r12,EX_TLB_SIZE; \
	105	std r15,EX_TLB_R15(r12); \
	106	mfspr r15,SPRN_SRR1; \
	107	std r16,EX_TLB_R16(r12); \
	108	mfspr r16,SPRN_SRR0; \
	109	std r10,EX_TLB_CR(r12); \
	110	std r11,EX_TLB_R12(r12); \
	111	mtspr SPRN_SPRG_TLB_EXFRAME,r14; \
	112	std r15,EX_TLB_SRR1(r12); \
	113	std r16,EX_TLB_SRR0(r12); \
	114	TLB_MISS_PROLOG_STATS
	115
	116	/* And these are the matching epilogs that restores things
	117	*
	118	* There are 3 epilogs:
	119	*
	120	* - SUCCESS : Unwinds one level
	121	* - ERROR : restore from level 0 and reset
	122	* - ERROR_SPECIAL : restore from current level and reset
	123	*
	124	* Normal errors use ERROR, that is, they restore the initial fault context
	125	* and trigger a fault. However, there is a special case for linear mapping
	126	* errors. Those should basically never happen, but if they do happen, we
	127	* want the error to point out the context that did that linear mapping
	128	* fault, not the initial level 0 (basically, we got a bogus PGF or something
	129	* like that). For userland errors on the linear mapping, there is no
	130	* difference since those are always level 0 anyway
	131	*/
	132
	133	#define TLB_MISS_RESTORE(freg) \
	134	ld r14,EX_TLB_CR(r12); \
	135	ld r10,EX_TLB_R10(r12); \
	136	ld r15,EX_TLB_SRR0(r12); \
	137	ld r16,EX_TLB_SRR1(r12); \
	138	mtspr SPRN_SPRG_TLB_EXFRAME,freg; \
	139	ld r11,EX_TLB_R11(r12); \
140	mtcr r14; \
141	ld r13,EX_TLB_R13(r12); \
142	ld r14,EX_TLB_R14(r12); \
143	mtspr SPRN_SRR0,r15; \
144	ld r15,EX_TLB_R15(r12); \
145	mtspr SPRN_SRR1,r16; \
146	TLB_MISS_RESTORE_STATS \
147	ld r16,EX_TLB_R16(r12); \
148	ld r12,EX_TLB_R12(r12); \
149
150	#define TLB_MISS_EPILOG_SUCCESS \
151	TLB_MISS_RESTORE(r12)
152
153	#define TLB_MISS_EPILOG_ERROR \
154	addi r12,r13,PACA_EXTLB; \
155	TLB_MISS_RESTORE(r12)
156
157	#define TLB_MISS_EPILOG_ERROR_SPECIAL \
158	addi r11,r13,PACA_EXTLB; \
159	TLB_MISS_RESTORE(r11)
160
161	#ifdef CONFIG_BOOK3E_MMU_TLB_STATS
162	#define TLB_MISS_PROLOG_STATS \
163	mflr r10; \
164	std r8,EX_TLB_R8(r12); \
165	std r9,EX_TLB_R9(r12); \
166	std r10,EX_TLB_LR(r12);
167	#define TLB_MISS_RESTORE_STATS \
168	ld r16,EX_TLB_LR(r12); \
169	ld r9,EX_TLB_R9(r12); \
170	ld r8,EX_TLB_R8(r12); \
171	mtlr r16;
172	#define TLB_MISS_STATS_D(name) \
173	addi r9,r13,MMSTAT_DSTATS+name; \
b1576fec	174	bl tlb_stat_inc;
13363ab9 BH	175	#define TLB_MISS_STATS_I(name) \
13363ab9 BH	176	addi r9,r13,MMSTAT_ISTATS+name; \
b1576fec	177	bl tlb_stat_inc;
13363ab9 BH	178	#define TLB_MISS_STATS_X(name) \
	179	ld r8,PACA_EXTLB+EX_TLB_ESR(r13); \
	180	cmpdi cr2,r8,-1; \
	181	beq cr2,61f; \
	182	addi r9,r13,MMSTAT_DSTATS+name; \
	183	b 62f; \
	184	61: addi r9,r13,MMSTAT_ISTATS+name; \
b1576fec	185	62: bl tlb_stat_inc;
13363ab9	186	#define TLB_MISS_STATS_SAVE_INFO \
f67f4ef5 SW	187	std r14,EX_TLB_ESR(r12); /* save ESR */
	188	#define TLB_MISS_STATS_SAVE_INFO_BOLTED \
	189	std r14,PACA_EXTLB+EX_TLB_ESR(r13); /* save ESR */
13363ab9 BH	190	#else
	191	#define TLB_MISS_PROLOG_STATS
	192	#define TLB_MISS_RESTORE_STATS
f67f4ef5 SW	193	#define TLB_MISS_PROLOG_STATS_BOLTED
f67f4ef5 SW	194	#define TLB_MISS_RESTORE_STATS_BOLTED
13363ab9 BH	195	#define TLB_MISS_STATS_D(name)
	196	#define TLB_MISS_STATS_I(name)
	197	#define TLB_MISS_STATS_X(name)
	198	#define TLB_MISS_STATS_Y(name)
	199	#define TLB_MISS_STATS_SAVE_INFO
f67f4ef5	200	#define TLB_MISS_STATS_SAVE_INFO_BOLTED
13363ab9 BH	201	#endif
13363ab9 BH	202
4b98d9e7	203	#define SET_IVOR(vector_number, vector_offset) \
1cb6e064 TC	204	LOAD_REG_ADDR(r3,interrupt_base_book3e);\
1cb6e064 TC	205	ori r3,r3,vector_offset@l; \
4b98d9e7	206	mtspr SPRN_IVOR##vector_number,r3;
13363ab9	207
50e51c13 NP	208	#define RFI_TO_KERNEL \
	209	rfi
	210
	211	#define RFI_TO_USER \
	212	rfi
	213
13363ab9 BH	214	#endif /* _ASM_POWERPC_EXCEPTION_64E_H */
13363ab9 BH	215