projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
parisc: Move disable_sr_hashing_asm() into .text section
[linux-2.6-block.git] / arch / parisc / kernel / pacache.S
CommitLineData
660662f8 1/* SPDX-License-Identifier: GPL-2.0-or-later */
1da177e4
LT		 2/*
3 * PARISC TLB and cache flushing support
4 * Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
5 * Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
6 * Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
1da177e4
LT		 7 */
8
9/*
10 * NOTE: fdc,fic, and pdc instructions that use base register modification
11 * should only use index and base registers that are not shadowed,
12 * so that the fast path emulation in the non access miss handler
13 * can be used.
14 */
15
413059f2 16#ifdef CONFIG_64BIT
1da177e4
LT		 17 .level 2.0w
18#else
1da177e4
LT		 19 .level 2.0
20#endif
21
1da177e4 22#include <asm/psw.h>
896a3756 23#include <asm/assembly.h>
1da177e4 24#include <asm/cache.h>
88776c0e 25#include <asm/ldcw.h>
3847dab7 26#include <asm/alternative.h>
8e9e9844 27#include <linux/linkage.h>
2a03bb9e 28#include <linux/init.h>
65fddcfc 29#include <linux/pgtable.h>
1da177e4 30
2a03bb9e
HD		 31 .section .text.hot
32 .align 16
1da177e4 33
f39cce65 34ENTRY_CFI(flush_tlb_all_local)
1da177e4
LT		 35 /*
36 * The pitlbe and pdtlbe instructions should only be used to
37 * flush the entire tlb. Also, there needs to be no intervening
38 * tlb operations, e.g. tlb misses, so the operation needs
39 * to happen in real mode with all interruptions disabled.
40 */
41
896a3756 42 /* pcxt_ssm_bug - relied upon translation! PA 2.0 Arch. F-4 and F-5 */
2fd83038 43 rsm PSW_SM_I, %r19 /* save I-bit state */
896a3756 44 load32 PA(1f), %r1
1da177e4
LT		 45 nop
46 nop
47 nop
48 nop
49 nop
896a3756
GG		 50
51 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1da177e4
LT		 52 mtctl %r0, %cr17 /* Clear IIASQ tail */
53 mtctl %r0, %cr17 /* Clear IIASQ head */
1da177e4
LT		 54 mtctl %r1, %cr18 /* IIAOQ head */
55 ldo 4(%r1), %r1
56 mtctl %r1, %cr18 /* IIAOQ tail */
896a3756
GG		 57 load32 REAL_MODE_PSW, %r1
58 mtctl %r1, %ipsw
1da177e4
LT		 59 rfi
60 nop
61
2fd83038 621: load32 PA(cache_info), %r1
1da177e4
LT		 63
64 /* Flush Instruction Tlb */
65
69245c97 6688: LDREG ITLB_SID_BASE(%r1), %r20
1da177e4
LT		 67 LDREG ITLB_SID_STRIDE(%r1), %r21
68 LDREG ITLB_SID_COUNT(%r1), %r22
69 LDREG ITLB_OFF_BASE(%r1), %arg0
70 LDREG ITLB_OFF_STRIDE(%r1), %arg1
71 LDREG ITLB_OFF_COUNT(%r1), %arg2
72 LDREG ITLB_LOOP(%r1), %arg3
73
872f6deb 74 addib,COND(=) -1, %arg3, fitoneloop /* Preadjust and test */
1da177e4
LT		 75 movb,<,n %arg3, %r31, fitdone /* If loop < 0, skip */
76 copy %arg0, %r28 /* Init base addr */
77
78fitmanyloop: /* Loop if LOOP >= 2 */
79 mtsp %r20, %sr1
80 add %r21, %r20, %r20 /* increment space */
81 copy %arg2, %r29 /* Init middle loop count */
82
83fitmanymiddle: /* Loop if LOOP >= 2 */
872f6deb 84 addib,COND(>) -1, %r31, fitmanymiddle /* Adjusted inner loop decr */
5035b230 85 pitlbe %r0(%sr1, %r28)
1da177e4 86 pitlbe,m %arg1(%sr1, %r28) /* Last pitlbe and addr adjust */
872f6deb 87 addib,COND(>) -1, %r29, fitmanymiddle /* Middle loop decr */
1da177e4
LT		 88 copy %arg3, %r31 /* Re-init inner loop count */
89
90 movb,tr %arg0, %r28, fitmanyloop /* Re-init base addr */
872f6deb 91 addib,COND(<=),n -1, %r22, fitdone /* Outer loop count decr */
1da177e4
LT		 92
93fitoneloop: /* Loop if LOOP = 1 */
94 mtsp %r20, %sr1
95 copy %arg0, %r28 /* init base addr */
96 copy %arg2, %r29 /* init middle loop count */
97
98fitonemiddle: /* Loop if LOOP = 1 */
872f6deb 99 addib,COND(>) -1, %r29, fitonemiddle /* Middle loop count decr */
1da177e4
LT		 100 pitlbe,m %arg1(%sr1, %r28) /* pitlbe for one loop */
101
872f6deb 102 addib,COND(>) -1, %r22, fitoneloop /* Outer loop count decr */
1da177e4
LT		 103 add %r21, %r20, %r20 /* increment space */
104
105fitdone:
69245c97 106 ALTERNATIVE(88b, fitdone, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
1da177e4
LT		 107
108 /* Flush Data Tlb */
109
110 LDREG DTLB_SID_BASE(%r1), %r20
111 LDREG DTLB_SID_STRIDE(%r1), %r21
112 LDREG DTLB_SID_COUNT(%r1), %r22
113 LDREG DTLB_OFF_BASE(%r1), %arg0
114 LDREG DTLB_OFF_STRIDE(%r1), %arg1
115 LDREG DTLB_OFF_COUNT(%r1), %arg2
116 LDREG DTLB_LOOP(%r1), %arg3
117
872f6deb 118 addib,COND(=) -1, %arg3, fdtoneloop /* Preadjust and test */
1da177e4
LT		 119 movb,<,n %arg3, %r31, fdtdone /* If loop < 0, skip */
120 copy %arg0, %r28 /* Init base addr */
121
122fdtmanyloop: /* Loop if LOOP >= 2 */
123 mtsp %r20, %sr1
124 add %r21, %r20, %r20 /* increment space */
125 copy %arg2, %r29 /* Init middle loop count */
126
127fdtmanymiddle: /* Loop if LOOP >= 2 */
872f6deb 128 addib,COND(>) -1, %r31, fdtmanymiddle /* Adjusted inner loop decr */
5035b230 129 pdtlbe %r0(%sr1, %r28)
1da177e4 130 pdtlbe,m %arg1(%sr1, %r28) /* Last pdtlbe and addr adjust */
872f6deb 131 addib,COND(>) -1, %r29, fdtmanymiddle /* Middle loop decr */
1da177e4
LT		 132 copy %arg3, %r31 /* Re-init inner loop count */
133
134 movb,tr %arg0, %r28, fdtmanyloop /* Re-init base addr */
872f6deb 135 addib,COND(<=),n -1, %r22,fdtdone /* Outer loop count decr */
1da177e4
LT		 136
137fdtoneloop: /* Loop if LOOP = 1 */
138 mtsp %r20, %sr1
139 copy %arg0, %r28 /* init base addr */
140 copy %arg2, %r29 /* init middle loop count */
141
142fdtonemiddle: /* Loop if LOOP = 1 */
872f6deb 143 addib,COND(>) -1, %r29, fdtonemiddle /* Middle loop count decr */
1da177e4
LT		 144 pdtlbe,m %arg1(%sr1, %r28) /* pdtlbe for one loop */
145
872f6deb 146 addib,COND(>) -1, %r22, fdtoneloop /* Outer loop count decr */
1da177e4
LT		 147 add %r21, %r20, %r20 /* increment space */
148
1da177e4 149
896a3756
GG		 150fdtdone:
151 /*
152 * Switch back to virtual mode
153 */
154 /* pcxt_ssm_bug */
155 rsm PSW_SM_I, %r0
156 load32 2f, %r1
157 nop
158 nop
159 nop
160 nop
161 nop
1da177e4 162
896a3756 163 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1da177e4
LT		 164 mtctl %r0, %cr17 /* Clear IIASQ tail */
165 mtctl %r0, %cr17 /* Clear IIASQ head */
1da177e4
LT		 166 mtctl %r1, %cr18 /* IIAOQ head */
167 ldo 4(%r1), %r1
168 mtctl %r1, %cr18 /* IIAOQ tail */
896a3756
GG		 169 load32 KERNEL_PSW, %r1
170 or %r1, %r19, %r1 /* I-bit to state on entry */
171 mtctl %r1, %ipsw /* restore I-bit (entire PSW) */
1da177e4
LT		 172 rfi
173 nop
174
1752: bv %r0(%r2)
176 nop
a5ff2130
HD		 177
178 /*
179 * When running in qemu, drop whole flush_tlb_all_local function and
180 * replace by one pdtlbe instruction, for which QEMU will drop all
181 * local TLB entries.
182 */
1833: pdtlbe %r0(%sr1,%r0)
184 bv,n %r0(%r2)
185 ALTERNATIVE_CODE(flush_tlb_all_local, 2, ALT_COND_RUN_ON_QEMU, 3b)
f39cce65 186ENDPROC_CFI(flush_tlb_all_local)
1da177e4 187
1da177e4
LT		 188 .import cache_info,data
189
f39cce65 190ENTRY_CFI(flush_instruction_cache_local)
3847dab7 19188: load32 cache_info, %r1
1da177e4
LT		 192
193 /* Flush Instruction Cache */
194
195 LDREG ICACHE_BASE(%r1), %arg0
196 LDREG ICACHE_STRIDE(%r1), %arg1
197 LDREG ICACHE_COUNT(%r1), %arg2
198 LDREG ICACHE_LOOP(%r1), %arg3
6d2ddc2f
JDA		 199 rsm PSW_SM_I, %r22 /* No mmgt ops during loop*/
200 mtsp %r0, %sr1
872f6deb 201 addib,COND(=) -1, %arg3, fioneloop /* Preadjust and test */
1da177e4
LT		 202 movb,<,n %arg3, %r31, fisync /* If loop < 0, do sync */
203
204fimanyloop: /* Loop if LOOP >= 2 */
872f6deb 205 addib,COND(>) -1, %r31, fimanyloop /* Adjusted inner loop decr */
9b3b331d 206 fice %r0(%sr1, %arg0)
1da177e4
LT		 207 fice,m %arg1(%sr1, %arg0) /* Last fice and addr adjust */
208 movb,tr %arg3, %r31, fimanyloop /* Re-init inner loop count */
872f6deb 209 addib,COND(<=),n -1, %arg2, fisync /* Outer loop decr */
1da177e4
LT		 210
211fioneloop: /* Loop if LOOP = 1 */
6d2ddc2f
JDA		 212 /* Some implementations may flush with a single fice instruction */
213 cmpib,COND(>>=),n 15, %arg2, fioneloop2
214
215fioneloop1:
216 fice,m %arg1(%sr1, %arg0)
217 fice,m %arg1(%sr1, %arg0)
218 fice,m %arg1(%sr1, %arg0)
219 fice,m %arg1(%sr1, %arg0)
220 fice,m %arg1(%sr1, %arg0)
221 fice,m %arg1(%sr1, %arg0)
222 fice,m %arg1(%sr1, %arg0)
223 fice,m %arg1(%sr1, %arg0)
224 fice,m %arg1(%sr1, %arg0)
225 fice,m %arg1(%sr1, %arg0)
226 fice,m %arg1(%sr1, %arg0)
227 fice,m %arg1(%sr1, %arg0)
228 fice,m %arg1(%sr1, %arg0)
229 fice,m %arg1(%sr1, %arg0)
230 fice,m %arg1(%sr1, %arg0)
231 addib,COND(>) -16, %arg2, fioneloop1
232 fice,m %arg1(%sr1, %arg0)
233
234 /* Check if done */
235 cmpb,COND(=),n %arg2, %r0, fisync /* Predict branch taken */
236
237fioneloop2:
238 addib,COND(>) -1, %arg2, fioneloop2 /* Outer loop count decr */
1da177e4
LT		 239 fice,m %arg1(%sr1, %arg0) /* Fice for one loop */
240
241fisync:
242 sync
896a3756 243 mtsm %r22 /* restore I-bit */
3847dab7 24489: ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1da177e4
LT		 245 bv %r0(%r2)
246 nop
f39cce65 247ENDPROC_CFI(flush_instruction_cache_local)
1da177e4 248
1da177e4 249
8e9e9844 250 .import cache_info, data
f39cce65 251ENTRY_CFI(flush_data_cache_local)
3847dab7 25288: load32 cache_info, %r1
1da177e4
LT		 253
254 /* Flush Data Cache */
255
256 LDREG DCACHE_BASE(%r1), %arg0
257 LDREG DCACHE_STRIDE(%r1), %arg1
258 LDREG DCACHE_COUNT(%r1), %arg2
259 LDREG DCACHE_LOOP(%r1), %arg3
6d2ddc2f
JDA		 260 rsm PSW_SM_I, %r22 /* No mmgt ops during loop*/
261 mtsp %r0, %sr1
872f6deb 262 addib,COND(=) -1, %arg3, fdoneloop /* Preadjust and test */
1da177e4
LT		 263 movb,<,n %arg3, %r31, fdsync /* If loop < 0, do sync */
264
265fdmanyloop: /* Loop if LOOP >= 2 */
872f6deb 266 addib,COND(>) -1, %r31, fdmanyloop /* Adjusted inner loop decr */
9b3b331d 267 fdce %r0(%sr1, %arg0)
1da177e4
LT		 268 fdce,m %arg1(%sr1, %arg0) /* Last fdce and addr adjust */
269 movb,tr %arg3, %r31, fdmanyloop /* Re-init inner loop count */
872f6deb 270 addib,COND(<=),n -1, %arg2, fdsync /* Outer loop decr */
1da177e4
LT		 271
272fdoneloop: /* Loop if LOOP = 1 */
6d2ddc2f
JDA		 273 /* Some implementations may flush with a single fdce instruction */
274 cmpib,COND(>>=),n 15, %arg2, fdoneloop2
275
276fdoneloop1:
277 fdce,m %arg1(%sr1, %arg0)
278 fdce,m %arg1(%sr1, %arg0)
279 fdce,m %arg1(%sr1, %arg0)
280 fdce,m %arg1(%sr1, %arg0)
281 fdce,m %arg1(%sr1, %arg0)
282 fdce,m %arg1(%sr1, %arg0)
283 fdce,m %arg1(%sr1, %arg0)
284 fdce,m %arg1(%sr1, %arg0)
285 fdce,m %arg1(%sr1, %arg0)
286 fdce,m %arg1(%sr1, %arg0)
287 fdce,m %arg1(%sr1, %arg0)
288 fdce,m %arg1(%sr1, %arg0)
289 fdce,m %arg1(%sr1, %arg0)
290 fdce,m %arg1(%sr1, %arg0)
291 fdce,m %arg1(%sr1, %arg0)
292 addib,COND(>) -16, %arg2, fdoneloop1
293 fdce,m %arg1(%sr1, %arg0)
294
295 /* Check if done */
296 cmpb,COND(=),n %arg2, %r0, fdsync /* Predict branch taken */
297
298fdoneloop2:
299 addib,COND(>) -1, %arg2, fdoneloop2 /* Outer loop count decr */
1da177e4
LT		 300 fdce,m %arg1(%sr1, %arg0) /* Fdce for one loop */
301
302fdsync:
303 syncdma
304 sync
896a3756 305 mtsm %r22 /* restore I-bit */
3847dab7 30689: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1da177e4
LT		 307 bv %r0(%r2)
308 nop
f39cce65 309ENDPROC_CFI(flush_data_cache_local)
1da177e4 310
6d2ddc2f
JDA		 311/* Clear page using kernel mapping. */
312
f39cce65 313ENTRY_CFI(clear_page_asm)
6d2ddc2f
JDA		 314#ifdef CONFIG_64BIT
315
316 /* Unroll the loop. */
317 ldi (PAGE_SIZE / 128), %r1
318
3191:
320 std %r0, 0(%r26)
321 std %r0, 8(%r26)
322 std %r0, 16(%r26)
323 std %r0, 24(%r26)
324 std %r0, 32(%r26)
325 std %r0, 40(%r26)
326 std %r0, 48(%r26)
327 std %r0, 56(%r26)
328 std %r0, 64(%r26)
329 std %r0, 72(%r26)
330 std %r0, 80(%r26)
331 std %r0, 88(%r26)
332 std %r0, 96(%r26)
333 std %r0, 104(%r26)
334 std %r0, 112(%r26)
335 std %r0, 120(%r26)
336
337 /* Note reverse branch hint for addib is taken. */
338 addib,COND(>),n -1, %r1, 1b
339 ldo 128(%r26), %r26
340
341#else
342
343 /*
344 * Note that until (if) we start saving the full 64-bit register
345 * values on interrupt, we can't use std on a 32 bit kernel.
346 */
347 ldi (PAGE_SIZE / 64), %r1
348
3491:
350 stw %r0, 0(%r26)
351 stw %r0, 4(%r26)
352 stw %r0, 8(%r26)
353 stw %r0, 12(%r26)
354 stw %r0, 16(%r26)
355 stw %r0, 20(%r26)
356 stw %r0, 24(%r26)
357 stw %r0, 28(%r26)
358 stw %r0, 32(%r26)
359 stw %r0, 36(%r26)
360 stw %r0, 40(%r26)
361 stw %r0, 44(%r26)
362 stw %r0, 48(%r26)
363 stw %r0, 52(%r26)
364 stw %r0, 56(%r26)
365 stw %r0, 60(%r26)
366
367 addib,COND(>),n -1, %r1, 1b
368 ldo 64(%r26), %r26
369#endif
370 bv %r0(%r2)
371 nop
f39cce65 372ENDPROC_CFI(clear_page_asm)
6d2ddc2f
JDA		 373
374/* Copy page using kernel mapping. */
375
f39cce65 376ENTRY_CFI(copy_page_asm)
413059f2 377#ifdef CONFIG_64BIT
1da177e4
LT		 378 /* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
379 * Unroll the loop by hand and arrange insn appropriately.
6d2ddc2f
JDA		 380 * Prefetch doesn't improve performance on rp3440.
381 * GCC probably can do this just as well...
1da177e4
LT		 382 */
383
6ebeafff 384 ldi (PAGE_SIZE / 128), %r1
2fd83038 385
6d2ddc2f
JDA		 3861: ldd 0(%r25), %r19
387 ldd 8(%r25), %r20
1da177e4
LT		 388
389 ldd 16(%r25), %r21
390 ldd 24(%r25), %r22
391 std %r19, 0(%r26)
392 std %r20, 8(%r26)
393
394 ldd 32(%r25), %r19
395 ldd 40(%r25), %r20
396 std %r21, 16(%r26)
397 std %r22, 24(%r26)
398
399 ldd 48(%r25), %r21
400 ldd 56(%r25), %r22
401 std %r19, 32(%r26)
402 std %r20, 40(%r26)
403
404 ldd 64(%r25), %r19
405 ldd 72(%r25), %r20
406 std %r21, 48(%r26)
407 std %r22, 56(%r26)
408
409 ldd 80(%r25), %r21
410 ldd 88(%r25), %r22
411 std %r19, 64(%r26)
412 std %r20, 72(%r26)
413
414 ldd 96(%r25), %r19
415 ldd 104(%r25), %r20
416 std %r21, 80(%r26)
417 std %r22, 88(%r26)
418
419 ldd 112(%r25), %r21
420 ldd 120(%r25), %r22
6d2ddc2f 421 ldo 128(%r25), %r25
1da177e4
LT		 422 std %r19, 96(%r26)
423 std %r20, 104(%r26)
424
1da177e4
LT		 425 std %r21, 112(%r26)
426 std %r22, 120(%r26)
1da177e4 427
6d2ddc2f
JDA		 428 /* Note reverse branch hint for addib is taken. */
429 addib,COND(>),n -1, %r1, 1b
430 ldo 128(%r26), %r26
1da177e4
LT		 431
432#else
433
434 /*
435 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
436 * bundles (very restricted rules for bundling).
437 * Note that until (if) we start saving
438 * the full 64 bit register values on interrupt, we can't
439 * use ldd/std on a 32 bit kernel.
440 */
37318a3c 441 ldw 0(%r25), %r19
6ebeafff 442 ldi (PAGE_SIZE / 64), %r1
1da177e4
LT		 443
4441:
1da177e4
LT		 445 ldw 4(%r25), %r20
446 ldw 8(%r25), %r21
447 ldw 12(%r25), %r22
448 stw %r19, 0(%r26)
449 stw %r20, 4(%r26)
450 stw %r21, 8(%r26)
451 stw %r22, 12(%r26)
452 ldw 16(%r25), %r19
453 ldw 20(%r25), %r20
454 ldw 24(%r25), %r21
455 ldw 28(%r25), %r22
456 stw %r19, 16(%r26)
457 stw %r20, 20(%r26)
458 stw %r21, 24(%r26)
459 stw %r22, 28(%r26)
460 ldw 32(%r25), %r19
461 ldw 36(%r25), %r20
462 ldw 40(%r25), %r21
463 ldw 44(%r25), %r22
464 stw %r19, 32(%r26)
465 stw %r20, 36(%r26)
466 stw %r21, 40(%r26)
467 stw %r22, 44(%r26)
468 ldw 48(%r25), %r19
469 ldw 52(%r25), %r20
470 ldw 56(%r25), %r21
471 ldw 60(%r25), %r22
472 stw %r19, 48(%r26)
473 stw %r20, 52(%r26)
37318a3c 474 ldo 64(%r25), %r25
1da177e4
LT		 475 stw %r21, 56(%r26)
476 stw %r22, 60(%r26)
477 ldo 64(%r26), %r26
872f6deb 478 addib,COND(>),n -1, %r1, 1b
37318a3c 479 ldw 0(%r25), %r19
1da177e4
LT		 480#endif
481 bv %r0(%r2)
482 nop
f39cce65 483ENDPROC_CFI(copy_page_asm)
1da177e4
LT		 484
485/*
486 * NOTE: Code in clear_user_page has a hard coded dependency on the
487 * maximum alias boundary being 4 Mb. We've been assured by the
488 * parisc chip designers that there will not ever be a parisc
489 * chip with a larger alias boundary (Never say never :-) ).
490 *
491 * Subtle: the dtlb miss handlers support the temp alias region by
492 * "knowing" that if a dtlb miss happens within the temp alias
493 * region it must have occurred while in clear_user_page. Since
494 * this routine makes use of processor local translations, we
495 * don't want to insert them into the kernel page table. Instead,
496 * we load up some general registers (they need to be registers
497 * which aren't shadowed) with the physical page numbers (preshifted
498 * for tlb insertion) needed to insert the translations. When we
499 * miss on the translation, the dtlb miss handler inserts the
500 * translation into the tlb using these values:
501 *
502 * %r26 physical page (shifted for tlb insert) of "to" translation
503 * %r23 physical page (shifted for tlb insert) of "from" translation
504 */
505
6a45716a
HD		 506 /* Drop prot bits and convert to page addr for iitlbt and idtlbt */
507 #define PAGE_ADD_SHIFT (PAGE_SHIFT-12)
508 .macro convert_phys_for_tlb_insert20 phys
509 extrd,u \phys, 56-PAGE_ADD_SHIFT, 32-PAGE_ADD_SHIFT, \phys
510#if _PAGE_SIZE_ENCODING_DEFAULT
511 depdi _PAGE_SIZE_ENCODING_DEFAULT, 63, (63-58), \phys
512#endif
513 .endm
514
1da177e4 515 /*
910a8643
JDA		 516 * copy_user_page_asm() performs a page copy using mappings
517 * equivalent to the user page mappings. It can be used to
518 * implement copy_user_page() but unfortunately both the `from'
519 * and `to' pages need to be flushed through mappings equivalent
520 * to the user mappings after the copy because the kernel accesses
521 * the `from' page through the kmap kernel mapping and the `to'
522 * page needs to be flushed since code can be copied. As a
523 * result, this implementation is less efficient than the simpler
524 * copy using the kernel mapping. It only needs the `from' page
525 * to flushed via the user mapping. The kunmap routines handle
526 * the flushes needed for the kernel mapping.
1da177e4
LT		 527 *
528 * I'm still keeping this around because it may be possible to
529 * use it if more information is passed into copy_user_page().
530 * Have to do some measurements to see if it is worthwhile to
531 * lobby for such a change.
6d2ddc2f 532 *
1da177e4
LT		 533 */
534
f39cce65 535ENTRY_CFI(copy_user_page_asm)
6d2ddc2f
JDA		 536 /* Convert virtual `to' and `from' addresses to physical addresses.
537 Move `from' physical address to non shadowed register. */
1da177e4
LT		 538 ldil L%(__PAGE_OFFSET), %r1
539 sub %r26, %r1, %r26
6d2ddc2f 540 sub %r25, %r1, %r23
1da177e4
LT		 541
542 ldil L%(TMPALIAS_MAP_START), %r28
413059f2 543#ifdef CONFIG_64BIT
6d2ddc2f
JDA		 544#if (TMPALIAS_MAP_START >= 0x80000000)
545 depdi 0, 31,32, %r28 /* clear any sign extension */
546#endif
6a45716a
HD		 547 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
548 convert_phys_for_tlb_insert20 %r23 /* convert phys addr to tlb insert format */
6d2ddc2f 549 depd %r24,63,22, %r28 /* Form aliased virtual address 'to' */
d845b5fb 550 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
1da177e4
LT		 551 copy %r28, %r29
552 depdi 1, 41,1, %r29 /* Form aliased virtual address 'from' */
553#else
554 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
555 extrw,u %r23, 24,25, %r23 /* convert phys addr to tlb insert format */
556 depw %r24, 31,22, %r28 /* Form aliased virtual address 'to' */
d845b5fb 557 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
1da177e4
LT		 558 copy %r28, %r29
559 depwi 1, 9,1, %r29 /* Form aliased virtual address 'from' */
560#endif
561
562 /* Purge any old translations */
563
6d2ddc2f 564#ifdef CONFIG_PA20
5035b230
JDA		 565 pdtlb,l %r0(%r28)
566 pdtlb,l %r0(%r29)
6d2ddc2f 567#else
3847dab7
HD		 5680: pdtlb %r0(%r28)
5691: pdtlb %r0(%r29)
3847dab7
HD		 570 ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
571 ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SMP, INSN_PxTLB)
6d2ddc2f
JDA		 572#endif
573
574#ifdef CONFIG_64BIT
575 /* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
576 * Unroll the loop by hand and arrange insn appropriately.
577 * GCC probably can do this just as well.
578 */
1da177e4 579
6d2ddc2f
JDA		 580 ldd 0(%r29), %r19
581 ldi (PAGE_SIZE / 128), %r1
582
5831: ldd 8(%r29), %r20
584
585 ldd 16(%r29), %r21
586 ldd 24(%r29), %r22
587 std %r19, 0(%r28)
588 std %r20, 8(%r28)
589
590 ldd 32(%r29), %r19
591 ldd 40(%r29), %r20
592 std %r21, 16(%r28)
593 std %r22, 24(%r28)
594
595 ldd 48(%r29), %r21
596 ldd 56(%r29), %r22
597 std %r19, 32(%r28)
598 std %r20, 40(%r28)
599
600 ldd 64(%r29), %r19
601 ldd 72(%r29), %r20
602 std %r21, 48(%r28)
603 std %r22, 56(%r28)
604
605 ldd 80(%r29), %r21
606 ldd 88(%r29), %r22
607 std %r19, 64(%r28)
608 std %r20, 72(%r28)
609
610 ldd 96(%r29), %r19
611 ldd 104(%r29), %r20
612 std %r21, 80(%r28)
613 std %r22, 88(%r28)
614
615 ldd 112(%r29), %r21
616 ldd 120(%r29), %r22
617 std %r19, 96(%r28)
618 std %r20, 104(%r28)
619
620 ldo 128(%r29), %r29
621 std %r21, 112(%r28)
622 std %r22, 120(%r28)
623 ldo 128(%r28), %r28
624
625 /* conditional branches nullify on forward taken branch, and on
626 * non-taken backward branch. Note that .+4 is a backwards branch.
627 * The ldd should only get executed if the branch is taken.
628 */
629 addib,COND(>),n -1, %r1, 1b /* bundle 10 */
630 ldd 0(%r29), %r19 /* start next loads */
631
632#else
633 ldi (PAGE_SIZE / 64), %r1
1da177e4
LT		 634
635 /*
636 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
637 * bundles (very restricted rules for bundling). It probably
638 * does OK on PCXU and better, but we could do better with
639 * ldd/std instructions. Note that until (if) we start saving
640 * the full 64 bit register values on interrupt, we can't
641 * use ldd/std on a 32 bit kernel.
642 */
643
6d2ddc2f 6441: ldw 0(%r29), %r19
1da177e4
LT		 645 ldw 4(%r29), %r20
646 ldw 8(%r29), %r21
647 ldw 12(%r29), %r22
648 stw %r19, 0(%r28)
649 stw %r20, 4(%r28)
650 stw %r21, 8(%r28)
651 stw %r22, 12(%r28)
652 ldw 16(%r29), %r19
653 ldw 20(%r29), %r20
654 ldw 24(%r29), %r21
655 ldw 28(%r29), %r22
656 stw %r19, 16(%r28)
657 stw %r20, 20(%r28)
658 stw %r21, 24(%r28)
659 stw %r22, 28(%r28)
660 ldw 32(%r29), %r19
661 ldw 36(%r29), %r20
662 ldw 40(%r29), %r21
663 ldw 44(%r29), %r22
664 stw %r19, 32(%r28)
665 stw %r20, 36(%r28)
666 stw %r21, 40(%r28)
667 stw %r22, 44(%r28)
668 ldw 48(%r29), %r19
669 ldw 52(%r29), %r20
670 ldw 56(%r29), %r21
671 ldw 60(%r29), %r22
672 stw %r19, 48(%r28)
673 stw %r20, 52(%r28)
674 stw %r21, 56(%r28)
675 stw %r22, 60(%r28)
676 ldo 64(%r28), %r28
6d2ddc2f 677
872f6deb 678 addib,COND(>) -1, %r1,1b
1da177e4 679 ldo 64(%r29), %r29
6d2ddc2f 680#endif
1da177e4
LT		 681
682 bv %r0(%r2)
683 nop
f39cce65 684ENDPROC_CFI(copy_user_page_asm)
1da177e4 685
f39cce65 686ENTRY_CFI(clear_user_page_asm)
1da177e4
LT		 687 tophys_r1 %r26
688
689 ldil L%(TMPALIAS_MAP_START), %r28
413059f2 690#ifdef CONFIG_64BIT
1da177e4
LT		 691#if (TMPALIAS_MAP_START >= 0x80000000)
692 depdi 0, 31,32, %r28 /* clear any sign extension */
693#endif
6a45716a 694 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
1da177e4 695 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
6a45716a 696 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
1da177e4
LT		 697#else
698 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
699 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
d845b5fb 700 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
1da177e4
LT		 701#endif
702
703 /* Purge any old translation */
704
6d2ddc2f 705#ifdef CONFIG_PA20
5035b230 706 pdtlb,l %r0(%r28)
6d2ddc2f 707#else
3847dab7 7080: pdtlb %r0(%r28)
3847dab7 709 ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
6d2ddc2f 710#endif
1da177e4 711
413059f2 712#ifdef CONFIG_64BIT
6ebeafff 713 ldi (PAGE_SIZE / 128), %r1
1da177e4
LT		 714
715 /* PREFETCH (Write) has not (yet) been proven to help here */
2fd83038 716 /* #define PREFETCHW_OP ldd 256(%0), %r0 */
1da177e4
LT		 717
7181: std %r0, 0(%r28)
719 std %r0, 8(%r28)
720 std %r0, 16(%r28)
721 std %r0, 24(%r28)
722 std %r0, 32(%r28)
723 std %r0, 40(%r28)
724 std %r0, 48(%r28)
725 std %r0, 56(%r28)
726 std %r0, 64(%r28)
727 std %r0, 72(%r28)
728 std %r0, 80(%r28)
729 std %r0, 88(%r28)
730 std %r0, 96(%r28)
731 std %r0, 104(%r28)
732 std %r0, 112(%r28)
733 std %r0, 120(%r28)
872f6deb 734 addib,COND(>) -1, %r1, 1b
1da177e4
LT		 735 ldo 128(%r28), %r28
736
413059f2 737#else /* ! CONFIG_64BIT */
6ebeafff 738 ldi (PAGE_SIZE / 64), %r1
1da177e4 739
6d2ddc2f 7401: stw %r0, 0(%r28)
1da177e4
LT		 741 stw %r0, 4(%r28)
742 stw %r0, 8(%r28)
743 stw %r0, 12(%r28)
744 stw %r0, 16(%r28)
745 stw %r0, 20(%r28)
746 stw %r0, 24(%r28)
747 stw %r0, 28(%r28)
748 stw %r0, 32(%r28)
749 stw %r0, 36(%r28)
750 stw %r0, 40(%r28)
751 stw %r0, 44(%r28)
752 stw %r0, 48(%r28)
753 stw %r0, 52(%r28)
754 stw %r0, 56(%r28)
755 stw %r0, 60(%r28)
872f6deb 756 addib,COND(>) -1, %r1, 1b
1da177e4 757 ldo 64(%r28), %r28
413059f2 758#endif /* CONFIG_64BIT */
1da177e4
LT		 759
760 bv %r0(%r2)
761 nop
f39cce65 762ENDPROC_CFI(clear_user_page_asm)
1da177e4 763
f39cce65 764ENTRY_CFI(flush_dcache_page_asm)
f311847c
JB		 765 ldil L%(TMPALIAS_MAP_START), %r28
766#ifdef CONFIG_64BIT
767#if (TMPALIAS_MAP_START >= 0x80000000)
768 depdi 0, 31,32, %r28 /* clear any sign extension */
f311847c 769#endif
6a45716a 770 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
f311847c 771 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
6a45716a 772 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
f311847c
JB		 773#else
774 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
775 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
d845b5fb 776 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
f311847c
JB		 777#endif
778
779 /* Purge any old translation */
780
6d2ddc2f 781#ifdef CONFIG_PA20
5035b230 782 pdtlb,l %r0(%r28)
6d2ddc2f 783#else
3847dab7 7840: pdtlb %r0(%r28)
3847dab7 785 ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
6d2ddc2f 786#endif
f311847c 787
3847dab7 78888: ldil L%dcache_stride, %r1
d65ea48d 789 ldw R%dcache_stride(%r1), r31
1da177e4 790
413059f2 791#ifdef CONFIG_64BIT
1da177e4
LT		 792 depdi,z 1, 63-PAGE_SHIFT,1, %r25
793#else
794 depwi,z 1, 31-PAGE_SHIFT,1, %r25
795#endif
f311847c 796 add %r28, %r25, %r25
d65ea48d
JDA		 797 sub %r25, r31, %r25
798
4c5fe5db 7991: fdc,m r31(%r28)
d65ea48d
JDA		 800 fdc,m r31(%r28)
801 fdc,m r31(%r28)
802 fdc,m r31(%r28)
803 fdc,m r31(%r28)
804 fdc,m r31(%r28)
805 fdc,m r31(%r28)
806 fdc,m r31(%r28)
807 fdc,m r31(%r28)
808 fdc,m r31(%r28)
809 fdc,m r31(%r28)
810 fdc,m r31(%r28)
811 fdc,m r31(%r28)
812 fdc,m r31(%r28)
813 fdc,m r31(%r28)
4c5fe5db 814 cmpb,COND(>>) %r25, %r28, 1b /* predict taken */
d65ea48d 815 fdc,m r31(%r28)
1da177e4 816
3847dab7 81789: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1da177e4
LT		 818 sync
819 bv %r0(%r2)
6d2ddc2f 820 nop
f39cce65 821ENDPROC_CFI(flush_dcache_page_asm)
f311847c 822
4c5fe5db
JDA		 823ENTRY_CFI(purge_dcache_page_asm)
824 ldil L%(TMPALIAS_MAP_START), %r28
825#ifdef CONFIG_64BIT
826#if (TMPALIAS_MAP_START >= 0x80000000)
827 depdi 0, 31,32, %r28 /* clear any sign extension */
828#endif
829 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
830 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
831 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
832#else
833 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
834 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
835 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
836#endif
837
838 /* Purge any old translation */
839
840#ifdef CONFIG_PA20
841 pdtlb,l %r0(%r28)
842#else
4c5fe5db 8430: pdtlb %r0(%r28)
4c5fe5db
JDA		 844 ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
845#endif
846
84788: ldil L%dcache_stride, %r1
848 ldw R%dcache_stride(%r1), r31
849
850#ifdef CONFIG_64BIT
851 depdi,z 1, 63-PAGE_SHIFT,1, %r25
852#else
853 depwi,z 1, 31-PAGE_SHIFT,1, %r25
854#endif
855 add %r28, %r25, %r25
856 sub %r25, r31, %r25
857
8581: pdc,m r31(%r28)
859 pdc,m r31(%r28)
860 pdc,m r31(%r28)
861 pdc,m r31(%r28)
862 pdc,m r31(%r28)
863 pdc,m r31(%r28)
864 pdc,m r31(%r28)
865 pdc,m r31(%r28)
866 pdc,m r31(%r28)
867 pdc,m r31(%r28)
868 pdc,m r31(%r28)
869 pdc,m r31(%r28)
870 pdc,m r31(%r28)
871 pdc,m r31(%r28)
872 pdc,m r31(%r28)
873 cmpb,COND(>>) %r25, %r28, 1b /* predict taken */
874 pdc,m r31(%r28)
875
87689: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
877 sync
878 bv %r0(%r2)
879 nop
880ENDPROC_CFI(purge_dcache_page_asm)
881
f39cce65 882ENTRY_CFI(flush_icache_page_asm)
f311847c 883 ldil L%(TMPALIAS_MAP_START), %r28
413059f2 884#ifdef CONFIG_64BIT
f311847c
JB		 885#if (TMPALIAS_MAP_START >= 0x80000000)
886 depdi 0, 31,32, %r28 /* clear any sign extension */
f311847c 887#endif
6a45716a 888 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
f311847c 889 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
d845b5fb 890 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
1da177e4 891#else
f311847c
JB		 892 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
893 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
d845b5fb 894 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
1da177e4 895#endif
1da177e4 896
5035b230
JDA		 897 /* Purge any old translation. Note that the FIC instruction
898 * may use either the instruction or data TLB. Given that we
899 * have a flat address space, it's not clear which TLB will be
900 * used. So, we purge both entries. */
1da177e4 901
6d2ddc2f 902#ifdef CONFIG_PA20
5035b230 903 pdtlb,l %r0(%r28)
3847dab7
HD		 9041: pitlb,l %r0(%sr4,%r28)
905 ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
6d2ddc2f 906#else
3847dab7
HD		 9070: pdtlb %r0(%r28)
9081: pitlb %r0(%sr4,%r28)
3847dab7
HD		 909 ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
910 ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SMP, INSN_PxTLB)
911 ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
6d2ddc2f 912#endif
f311847c 913
3847dab7 91488: ldil L%icache_stride, %r1
d65ea48d 915 ldw R%icache_stride(%r1), %r31
f311847c
JB		 916
917#ifdef CONFIG_64BIT
918 depdi,z 1, 63-PAGE_SHIFT,1, %r25
919#else
920 depwi,z 1, 31-PAGE_SHIFT,1, %r25
921#endif
922 add %r28, %r25, %r25
d65ea48d 923 sub %r25, %r31, %r25
f311847c 924
207f583d
JDA		 925 /* fic only has the type 26 form on PA1.1, requiring an
926 * explicit space specification, so use %sr4 */
d65ea48d
JDA		 9271: fic,m %r31(%sr4,%r28)
928 fic,m %r31(%sr4,%r28)
929 fic,m %r31(%sr4,%r28)
930 fic,m %r31(%sr4,%r28)
931 fic,m %r31(%sr4,%r28)
932 fic,m %r31(%sr4,%r28)
933 fic,m %r31(%sr4,%r28)
934 fic,m %r31(%sr4,%r28)
935 fic,m %r31(%sr4,%r28)
936 fic,m %r31(%sr4,%r28)
937 fic,m %r31(%sr4,%r28)
938 fic,m %r31(%sr4,%r28)
939 fic,m %r31(%sr4,%r28)
940 fic,m %r31(%sr4,%r28)
941 fic,m %r31(%sr4,%r28)
4c5fe5db 942 cmpb,COND(>>) %r25, %r28, 1b /* predict taken */
d65ea48d 943 fic,m %r31(%sr4,%r28)
1da177e4 944
3847dab7 94589: ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1da177e4
LT		 946 sync
947 bv %r0(%r2)
6d2ddc2f 948 nop
f39cce65 949ENDPROC_CFI(flush_icache_page_asm)
1da177e4 950
f39cce65 951ENTRY_CFI(flush_kernel_dcache_page_asm)
3847dab7 95288: ldil L%dcache_stride, %r1
1da177e4
LT		 953 ldw R%dcache_stride(%r1), %r23
954
413059f2 955#ifdef CONFIG_64BIT
1da177e4
LT		 956 depdi,z 1, 63-PAGE_SHIFT,1, %r25
957#else
958 depwi,z 1, 31-PAGE_SHIFT,1, %r25
959#endif
960 add %r26, %r25, %r25
961 sub %r25, %r23, %r25
962
f311847c
JB		 9631: fdc,m %r23(%r26)
964 fdc,m %r23(%r26)
965 fdc,m %r23(%r26)
966 fdc,m %r23(%r26)
967 fdc,m %r23(%r26)
968 fdc,m %r23(%r26)
969 fdc,m %r23(%r26)
970 fdc,m %r23(%r26)
971 fdc,m %r23(%r26)
972 fdc,m %r23(%r26)
973 fdc,m %r23(%r26)
974 fdc,m %r23(%r26)
975 fdc,m %r23(%r26)
976 fdc,m %r23(%r26)
977 fdc,m %r23(%r26)
4c5fe5db 978 cmpb,COND(>>) %r25, %r26, 1b /* predict taken */
f311847c 979 fdc,m %r23(%r26)
1da177e4 980
3847dab7 98189: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1da177e4
LT		 982 sync
983 bv %r0(%r2)
984 nop
f39cce65 985ENDPROC_CFI(flush_kernel_dcache_page_asm)
1da177e4 986
f39cce65 987ENTRY_CFI(purge_kernel_dcache_page_asm)
3847dab7 98888: ldil L%dcache_stride, %r1
1da177e4
LT		 989 ldw R%dcache_stride(%r1), %r23
990
413059f2 991#ifdef CONFIG_64BIT
1da177e4
LT		 992 depdi,z 1, 63-PAGE_SHIFT,1, %r25
993#else
994 depwi,z 1, 31-PAGE_SHIFT,1, %r25
995#endif
996 add %r26, %r25, %r25
997 sub %r25, %r23, %r25
998
9991: pdc,m %r23(%r26)
1000 pdc,m %r23(%r26)
1001 pdc,m %r23(%r26)
1002 pdc,m %r23(%r26)
1003 pdc,m %r23(%r26)
1004 pdc,m %r23(%r26)
1005 pdc,m %r23(%r26)
1006 pdc,m %r23(%r26)
1007 pdc,m %r23(%r26)
1008 pdc,m %r23(%r26)
1009 pdc,m %r23(%r26)
1010 pdc,m %r23(%r26)
1011 pdc,m %r23(%r26)
1012 pdc,m %r23(%r26)
1013 pdc,m %r23(%r26)
4c5fe5db 1014 cmpb,COND(>>) %r25, %r26, 1b /* predict taken */
1da177e4
LT		 1015 pdc,m %r23(%r26)
1016
3847dab7 101789: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1da177e4
LT		 1018 sync
1019 bv %r0(%r2)
1020 nop
f39cce65 1021ENDPROC_CFI(purge_kernel_dcache_page_asm)
1da177e4 1022
f39cce65 1023ENTRY_CFI(flush_user_dcache_range_asm)
3847dab7 102488: ldil L%dcache_stride, %r1
1da177e4
LT		 1025 ldw R%dcache_stride(%r1), %r23
1026 ldo -1(%r23), %r21
1027 ANDCM %r26, %r21, %r26
1028
4c5fe5db
JDA		 1029#ifdef CONFIG_64BIT
1030 depd,z %r23, 59, 60, %r21
1031#else
1032 depw,z %r23, 27, 28, %r21
1033#endif
1034 add %r26, %r21, %r22
1035 cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
10361: add %r22, %r21, %r22
1037 fdc,m %r23(%sr3, %r26)
1038 fdc,m %r23(%sr3, %r26)
1039 fdc,m %r23(%sr3, %r26)
1040 fdc,m %r23(%sr3, %r26)
1041 fdc,m %r23(%sr3, %r26)
1042 fdc,m %r23(%sr3, %r26)
1043 fdc,m %r23(%sr3, %r26)
1044 fdc,m %r23(%sr3, %r26)
1045 fdc,m %r23(%sr3, %r26)
1046 fdc,m %r23(%sr3, %r26)
1047 fdc,m %r23(%sr3, %r26)
1048 fdc,m %r23(%sr3, %r26)
1049 fdc,m %r23(%sr3, %r26)
1050 fdc,m %r23(%sr3, %r26)
1051 fdc,m %r23(%sr3, %r26)
1052 cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
1053 fdc,m %r23(%sr3, %r26)
1054
10552: cmpb,COND(>>),n %r25, %r26, 2b
1da177e4
LT		 1056 fdc,m %r23(%sr3, %r26)
1057
3847dab7 105889: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1da177e4
LT		 1059 sync
1060 bv %r0(%r2)
1061 nop
f39cce65 1062ENDPROC_CFI(flush_user_dcache_range_asm)
1da177e4 1063
f39cce65 1064ENTRY_CFI(flush_kernel_dcache_range_asm)
3847dab7 106588: ldil L%dcache_stride, %r1
1da177e4
LT		 1066 ldw R%dcache_stride(%r1), %r23
1067 ldo -1(%r23), %r21
1068 ANDCM %r26, %r21, %r26
1069
4c5fe5db
JDA		 1070#ifdef CONFIG_64BIT
1071 depd,z %r23, 59, 60, %r21
1072#else
1073 depw,z %r23, 27, 28, %r21
1074#endif
1075 add %r26, %r21, %r22
1076 cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
10771: add %r22, %r21, %r22
1078 fdc,m %r23(%r26)
1079 fdc,m %r23(%r26)
1080 fdc,m %r23(%r26)
1081 fdc,m %r23(%r26)
1082 fdc,m %r23(%r26)
1083 fdc,m %r23(%r26)
1084 fdc,m %r23(%r26)
1085 fdc,m %r23(%r26)
1086 fdc,m %r23(%r26)
1087 fdc,m %r23(%r26)
1088 fdc,m %r23(%r26)
1089 fdc,m %r23(%r26)
1090 fdc,m %r23(%r26)
1091 fdc,m %r23(%r26)
1092 fdc,m %r23(%r26)
1093 cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
1094 fdc,m %r23(%r26)
1095
10962: cmpb,COND(>>),n %r25, %r26, 2b /* predict taken */
1da177e4
LT		 1097 fdc,m %r23(%r26)
1098
1099 sync
3847dab7 110089: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1da177e4
LT		 1101 syncdma
1102 bv %r0(%r2)
1103 nop
f39cce65 1104ENDPROC_CFI(flush_kernel_dcache_range_asm)
1da177e4 1105
0adb24e0 1106ENTRY_CFI(purge_kernel_dcache_range_asm)
3847dab7 110788: ldil L%dcache_stride, %r1
0adb24e0
JDA		 1108 ldw R%dcache_stride(%r1), %r23
1109 ldo -1(%r23), %r21
1110 ANDCM %r26, %r21, %r26
1111
4c5fe5db
JDA		 1112#ifdef CONFIG_64BIT
1113 depd,z %r23, 59, 60, %r21
1114#else
1115 depw,z %r23, 27, 28, %r21
1116#endif
1117 add %r26, %r21, %r22
1118 cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
11191: add %r22, %r21, %r22
1120 pdc,m %r23(%r26)
1121 pdc,m %r23(%r26)
1122 pdc,m %r23(%r26)
1123 pdc,m %r23(%r26)
1124 pdc,m %r23(%r26)
1125 pdc,m %r23(%r26)
1126 pdc,m %r23(%r26)
1127 pdc,m %r23(%r26)
1128 pdc,m %r23(%r26)
1129 pdc,m %r23(%r26)
1130 pdc,m %r23(%r26)
1131 pdc,m %r23(%r26)
1132 pdc,m %r23(%r26)
1133 pdc,m %r23(%r26)
1134 pdc,m %r23(%r26)
1135 cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
1136 pdc,m %r23(%r26)
1137
11382: cmpb,COND(>>),n %r25, %r26, 2b /* predict taken */
0adb24e0
JDA		 1139 pdc,m %r23(%r26)
1140
1141 sync
3847dab7 114289: ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
0adb24e0
JDA		 1143 syncdma
1144 bv %r0(%r2)
1145 nop
0adb24e0
JDA		 1146ENDPROC_CFI(purge_kernel_dcache_range_asm)
1147
f39cce65 1148ENTRY_CFI(flush_user_icache_range_asm)
3847dab7 114988: ldil L%icache_stride, %r1
1da177e4
LT		 1150 ldw R%icache_stride(%r1), %r23
1151 ldo -1(%r23), %r21
1152 ANDCM %r26, %r21, %r26
1153
4c5fe5db
JDA		 1154#ifdef CONFIG_64BIT
1155 depd,z %r23, 59, 60, %r21
1156#else
1157 depw,z %r23, 27, 28, %r21
1158#endif
1159 add %r26, %r21, %r22
1160 cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
11611: add %r22, %r21, %r22
1162 fic,m %r23(%sr3, %r26)
1163 fic,m %r23(%sr3, %r26)
1164 fic,m %r23(%sr3, %r26)
1165 fic,m %r23(%sr3, %r26)
1166 fic,m %r23(%sr3, %r26)
1167 fic,m %r23(%sr3, %r26)
1168 fic,m %r23(%sr3, %r26)
1169 fic,m %r23(%sr3, %r26)
1170 fic,m %r23(%sr3, %r26)
1171 fic,m %r23(%sr3, %r26)
1172 fic,m %r23(%sr3, %r26)
1173 fic,m %r23(%sr3, %r26)
1174 fic,m %r23(%sr3, %r26)
1175 fic,m %r23(%sr3, %r26)
1176 fic,m %r23(%sr3, %r26)
1177 cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
1178 fic,m %r23(%sr3, %r26)
1179
11802: cmpb,COND(>>),n %r25, %r26, 2b
1da177e4
LT		 1181 fic,m %r23(%sr3, %r26)
1182
3847dab7 118389: ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1da177e4
LT		 1184 sync
1185 bv %r0(%r2)
1186 nop
f39cce65 1187ENDPROC_CFI(flush_user_icache_range_asm)
1da177e4 1188
f39cce65 1189ENTRY_CFI(flush_kernel_icache_page)
3847dab7 119088: ldil L%icache_stride, %r1
1da177e4
LT		 1191 ldw R%icache_stride(%r1), %r23
1192
413059f2 1193#ifdef CONFIG_64BIT
1da177e4
LT		 1194 depdi,z 1, 63-PAGE_SHIFT,1, %r25
1195#else
1196 depwi,z 1, 31-PAGE_SHIFT,1, %r25
1197#endif
1198 add %r26, %r25, %r25
1199 sub %r25, %r23, %r25
1200
1201
e635c96e
MW		 12021: fic,m %r23(%sr4, %r26)
1203 fic,m %r23(%sr4, %r26)
1204 fic,m %r23(%sr4, %r26)
1205 fic,m %r23(%sr4, %r26)
1206 fic,m %r23(%sr4, %r26)
1207 fic,m %r23(%sr4, %r26)
1208 fic,m %r23(%sr4, %r26)
1209 fic,m %r23(%sr4, %r26)
1210 fic,m %r23(%sr4, %r26)
1211 fic,m %r23(%sr4, %r26)
1212 fic,m %r23(%sr4, %r26)
1213 fic,m %r23(%sr4, %r26)
1214 fic,m %r23(%sr4, %r26)
1215 fic,m %r23(%sr4, %r26)
1216 fic,m %r23(%sr4, %r26)
4c5fe5db 1217 cmpb,COND(>>) %r25, %r26, 1b /* predict taken */
e635c96e 1218 fic,m %r23(%sr4, %r26)
1da177e4 1219
3847dab7 122089: ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1da177e4
LT		 1221 sync
1222 bv %r0(%r2)
1223 nop
f39cce65 1224ENDPROC_CFI(flush_kernel_icache_page)
1da177e4 1225
f39cce65 1226ENTRY_CFI(flush_kernel_icache_range_asm)
3847dab7 122788: ldil L%icache_stride, %r1
1da177e4
LT		 1228 ldw R%icache_stride(%r1), %r23
1229 ldo -1(%r23), %r21
1230 ANDCM %r26, %r21, %r26
1231
4c5fe5db
JDA		 1232#ifdef CONFIG_64BIT
1233 depd,z %r23, 59, 60, %r21
1234#else
1235 depw,z %r23, 27, 28, %r21
1236#endif
1237 add %r26, %r21, %r22
1238 cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
12391: add %r22, %r21, %r22
1240 fic,m %r23(%sr4, %r26)
1241 fic,m %r23(%sr4, %r26)
1242 fic,m %r23(%sr4, %r26)
1243 fic,m %r23(%sr4, %r26)
1244 fic,m %r23(%sr4, %r26)
1245 fic,m %r23(%sr4, %r26)
1246 fic,m %r23(%sr4, %r26)
1247 fic,m %r23(%sr4, %r26)
1248 fic,m %r23(%sr4, %r26)
1249 fic,m %r23(%sr4, %r26)
1250 fic,m %r23(%sr4, %r26)
1251 fic,m %r23(%sr4, %r26)
1252 fic,m %r23(%sr4, %r26)
1253 fic,m %r23(%sr4, %r26)
1254 fic,m %r23(%sr4, %r26)
1255 cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
1256 fic,m %r23(%sr4, %r26)
1257
12582: cmpb,COND(>>),n %r25, %r26, 2b /* predict taken */
e635c96e 1259 fic,m %r23(%sr4, %r26)
1da177e4 1260
3847dab7 126189: ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1da177e4
LT		 1262 sync
1263 bv %r0(%r2)
1264 nop
f39cce65 1265ENDPROC_CFI(flush_kernel_icache_range_asm)
1da177e4 1266
4d7d4c3f 1267 .text
2a03bb9e 1268
896a3756
GG		 1269 /* align should cover use of rfi in disable_sr_hashing_asm and
1270 * srdis_done.
1271 */
1272 .align 256
f39cce65 1273ENTRY_CFI(disable_sr_hashing_asm)
896a3756
GG		 1274 /*
1275 * Switch to real mode
1276 */
1277 /* pcxt_ssm_bug */
1278 rsm PSW_SM_I, %r0
1279 load32 PA(1f), %r1
1da177e4
LT		 1280 nop
1281 nop
1282 nop
1283 nop
1284 nop
896a3756
GG		 1285
1286 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1da177e4
LT		 1287 mtctl %r0, %cr17 /* Clear IIASQ tail */
1288 mtctl %r0, %cr17 /* Clear IIASQ head */
1da177e4
LT		 1289 mtctl %r1, %cr18 /* IIAOQ head */
1290 ldo 4(%r1), %r1
1291 mtctl %r1, %cr18 /* IIAOQ tail */
896a3756
GG		 1292 load32 REAL_MODE_PSW, %r1
1293 mtctl %r1, %ipsw
1da177e4
LT		 1294 rfi
1295 nop
1296
12971: cmpib,=,n SRHASH_PCXST, %r26,srdis_pcxs
1298 cmpib,=,n SRHASH_PCXL, %r26,srdis_pcxl
1299 cmpib,=,n SRHASH_PA20, %r26,srdis_pa20
1300 b,n srdis_done
1301
1302srdis_pcxs:
1303
1304 /* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1305
1306 .word 0x141c1a00 /* mfdiag %dr0, %r28 */
1307 .word 0x141c1a00 /* must issue twice */
1308 depwi 0,18,1, %r28 /* Clear DHE (dcache hash enable) */
1309 depwi 0,20,1, %r28 /* Clear IHE (icache hash enable) */
1310 .word 0x141c1600 /* mtdiag %r28, %dr0 */
1311 .word 0x141c1600 /* must issue twice */
1312 b,n srdis_done
1313
1314srdis_pcxl:
1315
1316 /* Disable Space Register Hashing for PCXL */
1317
1318 .word 0x141c0600 /* mfdiag %dr0, %r28 */
1319 depwi 0,28,2, %r28 /* Clear DHASH_EN & IHASH_EN */
1320 .word 0x141c0240 /* mtdiag %r28, %dr0 */
1321 b,n srdis_done
1322
1323srdis_pa20:
1324
896a3756 1325 /* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
1da177e4
LT		 1326
1327 .word 0x144008bc /* mfdiag %dr2, %r28 */
1328 depdi 0, 54,1, %r28 /* clear DIAG_SPHASH_ENAB (bit 54) */
1329 .word 0x145c1840 /* mtdiag %r28, %dr2 */
1330
1da177e4 1331
896a3756 1332srdis_done:
1da177e4 1333 /* Switch back to virtual mode */
896a3756
GG		 1334 rsm PSW_SM_I, %r0 /* prep to load iia queue */
1335 load32 2f, %r1
1336 nop
1337 nop
1338 nop
1339 nop
1340 nop
1da177e4 1341
896a3756 1342 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1da177e4
LT		 1343 mtctl %r0, %cr17 /* Clear IIASQ tail */
1344 mtctl %r0, %cr17 /* Clear IIASQ head */
1da177e4
LT		 1345 mtctl %r1, %cr18 /* IIAOQ head */
1346 ldo 4(%r1), %r1
1347 mtctl %r1, %cr18 /* IIAOQ tail */
896a3756
GG		 1348 load32 KERNEL_PSW, %r1
1349 mtctl %r1, %ipsw
1da177e4
LT		 1350 rfi
1351 nop
1352
13532: bv %r0(%r2)
1354 nop
f39cce65 1355ENDPROC_CFI(disable_sr_hashing_asm)
1da177e4
LT		 1356
1357 .end
Linux 6.x block layer and io_uring tree(s)