[linux-2.6-block.git] / arch / powerpc / platforms / powermac / cache.S

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * This file contains low-level cache management functions
 * used for sleep and CPU speed changes on Apple machines.
 * (In fact the only thing that is Apple-specific is that we assume
 * that we can read from ROM at physical address 0xfff00000.)
 *
 *    Copyright (C) 2004 Paul Mackerras (paulus@samba.org) and
 *                       Benjamin Herrenschmidt (benh@kernel.crashing.org)
 */

#include <asm/processor.h>
#include <asm/ppc_asm.h>
#include <asm/cputable.h>
#include <asm/feature-fixups.h>

/*
 * Flush and disable all data caches (dL1, L2, L3). This is used
 * when going to sleep, when doing a PMU based cpufreq transition,
 * or when "offlining" a CPU on SMP machines. This code is over
 * paranoid, but I've had enough issues with various CPU revs and
 * bugs that I decided it was worth being over cautious
 */

_GLOBAL(flush_disable_caches)
#ifndef CONFIG_PPC_BOOK3S_32
	blr
#else
BEGIN_FTR_SECTION
	b	flush_disable_745x
END_FTR_SECTION_IFSET(CPU_FTR_SPEC7450)
BEGIN_FTR_SECTION
	b	flush_disable_75x
END_FTR_SECTION_IFSET(CPU_FTR_L2CR)
	b	__flush_disable_L1

/* This is the code for G3 and 74[01]0 */
flush_disable_75x:
	mflr	r10

	/* Turn off EE and DR in MSR */
	mfmsr	r11
	rlwinm	r0,r11,0,~MSR_EE
	rlwinm	r0,r0,0,~MSR_DR
	sync
	mtmsr	r0
	isync

	/* Stop DST streams */
BEGIN_FTR_SECTION
	DSSALL
	sync
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)

	/* Stop DPM */
	mfspr	r8,SPRN_HID0		/* Save SPRN_HID0 in r8 */
	rlwinm	r4,r8,0,12,10		/* Turn off HID0[DPM] */
	sync
	mtspr	SPRN_HID0,r4		/* Disable DPM */
	sync

	/* Disp-flush L1. We have a weird problem here that I never
	 * totally figured out. On 750FX, using the ROM for the flush
	 * results in a non-working flush. We use that workaround for
	 * now until I finally understand what's going on. --BenH
	 */

	/* ROM base by default */
	lis	r4,0xfff0
	mfpvr	r3
	srwi	r3,r3,16
	cmplwi	cr0,r3,0x7000
	bne+	1f
	/* RAM base on 750FX */
	li	r4,0
1:	li	r4,0x4000
	mtctr	r4
1:	lwz	r0,0(r4)
	addi	r4,r4,32
	bdnz	1b
	sync
	isync

	/* Disable / invalidate / enable L1 data */
	mfspr	r3,SPRN_HID0
	rlwinm	r3,r3,0,~(HID0_DCE | HID0_ICE)
	mtspr	SPRN_HID0,r3
	sync
	isync
	ori	r3,r3,(HID0_DCE|HID0_DCI|HID0_ICE|HID0_ICFI)
	sync
	isync
	mtspr	SPRN_HID0,r3
	xori	r3,r3,(HID0_DCI|HID0_ICFI)
	mtspr	SPRN_HID0,r3
	sync

	/* Get the current enable bit of the L2CR into r4 */
	mfspr	r5,SPRN_L2CR
	/* Set to data-only (pre-745x bit) */
	oris	r3,r5,L2CR_L2DO@h
	b	2f
	/* When disabling L2, code must be in L1 */
	.balign 32
1:	mtspr	SPRN_L2CR,r3
3:	sync
	isync
	b	1f
2:	b	3f
3:	sync
	isync
	b	1b
1:	/* disp-flush L2. The interesting thing here is that the L2 can be
	 * up to 2Mb ... so using the ROM, we'll end up wrapping back to memory
	 * but that is probbaly fine. We disp-flush over 4Mb to be safe
	 */
	lis	r4,2
	mtctr	r4
	lis	r4,0xfff0
1:	lwz	r0,0(r4)
	addi	r4,r4,32
	bdnz	1b
	sync
	isync
	lis	r4,2
	mtctr	r4
	lis	r4,0xfff0
1:	dcbf	0,r4
	addi	r4,r4,32
	bdnz	1b
	sync
	isync

	/* now disable L2 */
	rlwinm	r5,r5,0,~L2CR_L2E
	b	2f
	/* When disabling L2, code must be in L1 */
	.balign 32
1:	mtspr	SPRN_L2CR,r5
3:	sync
	isync
	b	1f
2:	b	3f
3:	sync
	isync
	b	1b
1:	sync
	isync
	/* Invalidate L2. This is pre-745x, we clear the L2I bit ourselves */
	oris	r4,r5,L2CR_L2I@h
	mtspr	SPRN_L2CR,r4
	sync
	isync

	/* Wait for the invalidation to complete */
1:	mfspr	r3,SPRN_L2CR
	rlwinm.	r0,r3,0,31,31
	bne	1b

	/* Clear L2I */
	xoris	r4,r4,L2CR_L2I@h
	sync
	mtspr	SPRN_L2CR,r4
	sync

	/* now disable the L1 data cache */
	mfspr	r0,SPRN_HID0
	rlwinm	r0,r0,0,~(HID0_DCE|HID0_ICE)
	mtspr	SPRN_HID0,r0
	sync
	isync

	/* Restore HID0[DPM] to whatever it was before */
	sync
	mfspr	r0,SPRN_HID0
	rlwimi	r0,r8,0,11,11		/* Turn back HID0[DPM] */
	mtspr	SPRN_HID0,r0
	sync

	/* restore DR and EE */
	sync
	mtmsr	r11
	isync

	mtlr	r10
	blr

/* This code is for 745x processors */
flush_disable_745x:
	/* Turn off EE and DR in MSR */
	mfmsr	r11
	rlwinm	r0,r11,0,~MSR_EE
	rlwinm	r0,r0,0,~MSR_DR
	sync
	mtmsr	r0
	isync

	/* Stop prefetch streams */
	DSSALL
	sync

	/* Disable L2 prefetching */
	mfspr	r0,SPRN_MSSCR0
	rlwinm	r0,r0,0,0,29
	mtspr	SPRN_MSSCR0,r0
	sync
	isync
	lis	r4,0
	dcbf	0,r4
	dcbf	0,r4
	dcbf	0,r4
	dcbf	0,r4
	dcbf	0,r4
	dcbf	0,r4
	dcbf	0,r4
	dcbf	0,r4

	/* Due to a bug with the HW flush on some CPU revs, we occasionally
	 * experience data corruption. I'm adding a displacement flush along
	 * with a dcbf loop over a few Mb to "help". The problem isn't totally
	 * fixed by this in theory, but at least, in practice, I couldn't reproduce
	 * it even with a big hammer...
	 */

        lis     r4,0x0002
        mtctr   r4
 	li      r4,0
1:
        lwz     r0,0(r4)
        addi    r4,r4,32                /* Go to start of next cache line */
        bdnz    1b
        isync

        /* Now, flush the first 4MB of memory */
        lis     r4,0x0002
        mtctr   r4
	li      r4,0
        sync
1:
        dcbf    0,r4
        addi    r4,r4,32                /* Go to start of next cache line */
        bdnz    1b

	/* Flush and disable the L1 data cache */
	mfspr	r6,SPRN_LDSTCR
	lis	r3,0xfff0	/* read from ROM for displacement flush */
	li	r4,0xfe		/* start with only way 0 unlocked */
	li	r5,128		/* 128 lines in each way */
1:	mtctr	r5
	rlwimi	r6,r4,0,24,31
	mtspr	SPRN_LDSTCR,r6
	sync
	isync
2:	lwz	r0,0(r3)	/* touch each cache line */
	addi	r3,r3,32
	bdnz	2b
	rlwinm	r4,r4,1,24,30	/* move on to the next way */
	ori	r4,r4,1
	cmpwi	r4,0xff		/* all done? */
	bne	1b
	/* now unlock the L1 data cache */
	li	r4,0
	rlwimi	r6,r4,0,24,31
	sync
	mtspr	SPRN_LDSTCR,r6
	sync
	isync

	/* Flush the L2 cache using the hardware assist */
	mfspr	r3,SPRN_L2CR
	cmpwi	r3,0		/* check if it is enabled first */
	bge	4f
	oris	r0,r3,(L2CR_L2IO_745x|L2CR_L2DO_745x)@h
	b	2f
	/* When disabling/locking L2, code must be in L1 */
	.balign 32
1:	mtspr	SPRN_L2CR,r0	/* lock the L2 cache */
3:	sync
	isync
	b	1f
2:	b	3f
3:	sync
	isync
	b	1b
1:	sync
	isync
	ori	r0,r3,L2CR_L2HWF_745x
	sync
	mtspr	SPRN_L2CR,r0	/* set the hardware flush bit */
3:	mfspr	r0,SPRN_L2CR	/* wait for it to go to 0 */
	andi.	r0,r0,L2CR_L2HWF_745x
	bne	3b
	sync
	rlwinm	r3,r3,0,~L2CR_L2E
	b	2f
	/* When disabling L2, code must be in L1 */
	.balign 32
1:	mtspr	SPRN_L2CR,r3	/* disable the L2 cache */
3:	sync
	isync
	b	1f
2:	b	3f
3:	sync
	isync
	b	1b
1:	sync
	isync
	oris	r4,r3,L2CR_L2I@h
	mtspr	SPRN_L2CR,r4
	sync
	isync
1:	mfspr	r4,SPRN_L2CR
	andis.	r0,r4,L2CR_L2I@h
	bne	1b
	sync

BEGIN_FTR_SECTION
	/* Flush the L3 cache using the hardware assist */
4:	mfspr	r3,SPRN_L3CR
	cmpwi	r3,0		/* check if it is enabled */
	bge	6f
	oris	r0,r3,L3CR_L3IO@h
	ori	r0,r0,L3CR_L3DO
	sync
	mtspr	SPRN_L3CR,r0	/* lock the L3 cache */
	sync
	isync
	ori	r0,r0,L3CR_L3HWF
	sync
	mtspr	SPRN_L3CR,r0	/* set the hardware flush bit */
5:	mfspr	r0,SPRN_L3CR	/* wait for it to go to zero */
	andi.	r0,r0,L3CR_L3HWF
	bne	5b
	rlwinm	r3,r3,0,~L3CR_L3E
	sync
	mtspr	SPRN_L3CR,r3	/* disable the L3 cache */
	sync
	ori	r4,r3,L3CR_L3I
	mtspr	SPRN_L3CR,r4
1:	mfspr	r4,SPRN_L3CR
	andi.	r0,r4,L3CR_L3I
	bne	1b
	sync
END_FTR_SECTION_IFSET(CPU_FTR_L3CR)

6:	mfspr	r0,SPRN_HID0	/* now disable the L1 data cache */
	rlwinm	r0,r0,0,~HID0_DCE
	mtspr	SPRN_HID0,r0
	sync
	isync
	mtmsr	r11		/* restore DR and EE */
	isync
	blr
#endif	/* CONFIG_PPC_BOOK3S_32 */
Commit	Line	Data
2874c5fd	1	/* SPDX-License-Identifier: GPL-2.0-or-later */
14cf11af PM	2	/*
	3	* This file contains low-level cache management functions
	4	* used for sleep and CPU speed changes on Apple machines.
	5	* (In fact the only thing that is Apple-specific is that we assume
	6	* that we can read from ROM at physical address 0xfff00000.)
	7	*
	8	* Copyright (C) 2004 Paul Mackerras (paulus@samba.org) and
	9	* Benjamin Herrenschmidt (benh@kernel.crashing.org)
14cf11af PM	10	*/
14cf11af PM	11
14cf11af PM	12	#include <asm/processor.h>
	13	#include <asm/ppc_asm.h>
	14	#include <asm/cputable.h>
2c86cd18	15	#include <asm/feature-fixups.h>
14cf11af PM	16
	17	/*
	18	* Flush and disable all data caches (dL1, L2, L3). This is used
	19	* when going to sleep, when doing a PMU based cpufreq transition,
	20	* or when "offlining" a CPU on SMP machines. This code is over
	21	* paranoid, but I've had enough issues with various CPU revs and
446957ba	22	* bugs that I decided it was worth being over cautious
14cf11af PM	23	*/
	24
	25	_GLOBAL(flush_disable_caches)
d7cceda9	26	#ifndef CONFIG_PPC_BOOK3S_32
14cf11af PM	27	blr
	28	#else
	29	BEGIN_FTR_SECTION
	30	b flush_disable_745x
	31	END_FTR_SECTION_IFSET(CPU_FTR_SPEC7450)
	32	BEGIN_FTR_SECTION
	33	b flush_disable_75x
	34	END_FTR_SECTION_IFSET(CPU_FTR_L2CR)
	35	b __flush_disable_L1
	36
	37	/* This is the code for G3 and 74[01]0 */
	38	flush_disable_75x:
	39	mflr r10
	40
	41	/* Turn off EE and DR in MSR */
	42	mfmsr r11
	43	rlwinm r0,r11,0,~MSR_EE
	44	rlwinm r0,r0,0,~MSR_DR
	45	sync
	46	mtmsr r0
	47	isync
	48
	49	/* Stop DST streams */
	50	BEGIN_FTR_SECTION
	51	DSSALL
	52	sync
	53	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
	54
	55	/* Stop DPM */
	56	mfspr r8,SPRN_HID0 /* Save SPRN_HID0 in r8 */
	57	rlwinm r4,r8,0,12,10 /* Turn off HID0[DPM] */
	58	sync
	59	mtspr SPRN_HID0,r4 /* Disable DPM */
	60	sync
	61
	62	/* Disp-flush L1. We have a weird problem here that I never
	63	* totally figured out. On 750FX, using the ROM for the flush
	64	* results in a non-working flush. We use that workaround for
	65	* now until I finally understand what's going on. --BenH
	66	*/
	67
	68	/* ROM base by default */
	69	lis r4,0xfff0
	70	mfpvr r3
	71	srwi r3,r3,16
	72	cmplwi cr0,r3,0x7000
	73	bne+ 1f
	74	/* RAM base on 750FX */
	75	li r4,0
	76	1: li r4,0x4000
	77	mtctr r4
	78	1: lwz r0,0(r4)
	79	addi r4,r4,32
	80	bdnz 1b
	81	sync
	82	isync
	83
	84	/* Disable / invalidate / enable L1 data */
	85	mfspr r3,SPRN_HID0
	86	rlwinm r3,r3,0,~(HID0_DCE \| HID0_ICE)
	87	mtspr SPRN_HID0,r3
	88	sync
	89	isync
	90	ori r3,r3,(HID0_DCE\|HID0_DCI\|HID0_ICE\|HID0_ICFI)
91	sync
92	isync
93	mtspr SPRN_HID0,r3
94	xori r3,r3,(HID0_DCI\|HID0_ICFI)
95	mtspr SPRN_HID0,r3
96	sync
97
98	/* Get the current enable bit of the L2CR into r4 */
99	mfspr r5,SPRN_L2CR
100	/* Set to data-only (pre-745x bit) */
101	oris r3,r5,L2CR_L2DO@h
102	b 2f
103	/* When disabling L2, code must be in L1 */
104	.balign 32
105	1: mtspr SPRN_L2CR,r3
106	3: sync
107	isync
108	b 1f
109	2: b 3f
110	3: sync
111	isync
112	b 1b
113	1: /* disp-flush L2. The interesting thing here is that the L2 can be
114	* up to 2Mb ... so using the ROM, we'll end up wrapping back to memory
115	* but that is probbaly fine. We disp-flush over 4Mb to be safe
116	*/
117	lis r4,2
118	mtctr r4
119	lis r4,0xfff0
120	1: lwz r0,0(r4)
121	addi r4,r4,32
122	bdnz 1b
123	sync
124	isync
125	lis r4,2
126	mtctr r4
127	lis r4,0xfff0
128	1: dcbf 0,r4
129	addi r4,r4,32
130	bdnz 1b
131	sync
132	isync
133
134	/* now disable L2 */
135	rlwinm r5,r5,0,~L2CR_L2E
136	b 2f
137	/* When disabling L2, code must be in L1 */
138	.balign 32
139	1: mtspr SPRN_L2CR,r5
140	3: sync
141	isync
142	b 1f
143	2: b 3f
144	3: sync
145	isync
146	b 1b
147	1: sync
148	isync
149	/* Invalidate L2. This is pre-745x, we clear the L2I bit ourselves */
150	oris r4,r5,L2CR_L2I@h
151	mtspr SPRN_L2CR,r4
152	sync
153	isync
154
155	/* Wait for the invalidation to complete */
156	1: mfspr r3,SPRN_L2CR
157	rlwinm. r0,r3,0,31,31
158	bne 1b
159
160	/* Clear L2I */
161	xoris r4,r4,L2CR_L2I@h
162	sync
163	mtspr SPRN_L2CR,r4
164	sync
165
166	/* now disable the L1 data cache */
167	mfspr r0,SPRN_HID0
168	rlwinm r0,r0,0,~(HID0_DCE\|HID0_ICE)
169	mtspr SPRN_HID0,r0
170	sync
171	isync
172
173	/* Restore HID0[DPM] to whatever it was before */
174	sync
175	mfspr r0,SPRN_HID0
176	rlwimi r0,r8,0,11,11 /* Turn back HID0[DPM] */
177	mtspr SPRN_HID0,r0
178	sync
179
180	/* restore DR and EE */
181	sync
182	mtmsr r11
183	isync
184
185	mtlr r10
186	blr
187
188	/* This code is for 745x processors */
189	flush_disable_745x:
190	/* Turn off EE and DR in MSR */
191	mfmsr r11
192	rlwinm r0,r11,0,~MSR_EE
193	rlwinm r0,r0,0,~MSR_DR
194	sync
195	mtmsr r0
196	isync
197
198	/* Stop prefetch streams */
199	DSSALL
200	sync
201
202	/* Disable L2 prefetching */
203	mfspr r0,SPRN_MSSCR0
204	rlwinm r0,r0,0,0,29
205	mtspr SPRN_MSSCR0,r0
206	sync
207	isync
208	lis r4,0
209	dcbf 0,r4
210	dcbf 0,r4
211	dcbf 0,r4
212	dcbf 0,r4
213	dcbf 0,r4
214	dcbf 0,r4
215	dcbf 0,r4
216	dcbf 0,r4
217
218	/* Due to a bug with the HW flush on some CPU revs, we occasionally
219	* experience data corruption. I'm adding a displacement flush along
220	* with a dcbf loop over a few Mb to "help". The problem isn't totally
221	* fixed by this in theory, but at least, in practice, I couldn't reproduce
222	* it even with a big hammer...
223	*/
224
225	lis r4,0x0002
226	mtctr r4
227	li r4,0
228	1:
229	lwz r0,0(r4)
230	addi r4,r4,32 /* Go to start of next cache line */
231	bdnz 1b
232	isync
233
234	/* Now, flush the first 4MB of memory */
235	lis r4,0x0002
236	mtctr r4
237	li r4,0
238	sync
239	1:
240	dcbf 0,r4
241	addi r4,r4,32 /* Go to start of next cache line */
242	bdnz 1b
243
244	/* Flush and disable the L1 data cache */
245	mfspr r6,SPRN_LDSTCR
246	lis r3,0xfff0 /* read from ROM for displacement flush */
247	li r4,0xfe /* start with only way 0 unlocked */
248	li r5,128 /* 128 lines in each way */
249	1: mtctr r5
250	rlwimi r6,r4,0,24,31
251	mtspr SPRN_LDSTCR,r6
252	sync
253	isync
254	2: lwz r0,0(r3) /* touch each cache line */
255	addi r3,r3,32
256	bdnz 2b
257	rlwinm r4,r4,1,24,30 /* move on to the next way */
258	ori r4,r4,1
259	cmpwi r4,0xff /* all done? */
260	bne 1b
261	/* now unlock the L1 data cache */
262	li r4,0
263	rlwimi r6,r4,0,24,31
264	sync
265	mtspr SPRN_LDSTCR,r6
266	sync
267	isync
268
269	/* Flush the L2 cache using the hardware assist */
270	mfspr r3,SPRN_L2CR
271	cmpwi r3,0 /* check if it is enabled first */
272	bge 4f
273	oris r0,r3,(L2CR_L2IO_745x\|L2CR_L2DO_745x)@h
274	b 2f
275	/* When disabling/locking L2, code must be in L1 */
276	.balign 32
277	1: mtspr SPRN_L2CR,r0 /* lock the L2 cache */
278	3: sync
279	isync
280	b 1f
281	2: b 3f
282	3: sync
283	isync
284	b 1b
285	1: sync
286	isync
287	ori r0,r3,L2CR_L2HWF_745x
288	sync
289	mtspr SPRN_L2CR,r0 /* set the hardware flush bit */
290	3: mfspr r0,SPRN_L2CR /* wait for it to go to 0 */
291	andi. r0,r0,L2CR_L2HWF_745x
292	bne 3b
293	sync
294	rlwinm r3,r3,0,~L2CR_L2E
295	b 2f
296	/* When disabling L2, code must be in L1 */
297	.balign 32
298	1: mtspr SPRN_L2CR,r3 /* disable the L2 cache */
299	3: sync
300	isync
301	b 1f
302	2: b 3f
303	3: sync
304	isync
305	b 1b
306	1: sync
307	isync
308	oris r4,r3,L2CR_L2I@h
309	mtspr SPRN_L2CR,r4
310	sync
311	isync
312	1: mfspr r4,SPRN_L2CR
313	andis. r0,r4,L2CR_L2I@h
314	bne 1b
315	sync
316
317	BEGIN_FTR_SECTION
318	/* Flush the L3 cache using the hardware assist */
319	4: mfspr r3,SPRN_L3CR
320	cmpwi r3,0 /* check if it is enabled */
321	bge 6f
322	oris r0,r3,L3CR_L3IO@h
323	ori r0,r0,L3CR_L3DO
324	sync
325	mtspr SPRN_L3CR,r0 /* lock the L3 cache */
326	sync
327	isync
328	ori r0,r0,L3CR_L3HWF
329	sync
330	mtspr SPRN_L3CR,r0 /* set the hardware flush bit */
331	5: mfspr r0,SPRN_L3CR /* wait for it to go to zero */
332	andi. r0,r0,L3CR_L3HWF
333	bne 5b
334	rlwinm r3,r3,0,~L3CR_L3E
335	sync
336	mtspr SPRN_L3CR,r3 /* disable the L3 cache */
337	sync
338	ori r4,r3,L3CR_L3I
339	mtspr SPRN_L3CR,r4
340	1: mfspr r4,SPRN_L3CR
341	andi. r0,r4,L3CR_L3I
342	bne 1b
343	sync
344	END_FTR_SECTION_IFSET(CPU_FTR_L3CR)
345
346	6: mfspr r0,SPRN_HID0 /* now disable the L1 data cache */
347	rlwinm r0,r0,0,~HID0_DCE
348	mtspr SPRN_HID0,r0
349	sync
350	isync
351	mtmsr r11 /* restore DR and EE */
352	isync
353	blr
d7cceda9	354	#endif /* CONFIG_PPC_BOOK3S_32 */