projects / linux-block.git / blame
| Commit | Line | Data |
|---|---|---|
| 1a59d1b8 | 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
| fde69282 | 2 | /* |
| fde69282 AB |
3 | * |
| 4 | * Copyright (C) IBM Corporation, 2012 | |
| 5 | * | |
| 6 | * Author: Anton Blanchard <anton@au.ibm.com> | |
| 7 | */ | |
| 8 | #include <asm/page.h> | |
| 9 | #include <asm/ppc_asm.h> | |
| 10 | ||
| fde69282 AB |
11 | _GLOBAL(copypage_power7) |
| 12 | /* | |
| 13 | * We prefetch both the source and destination using enhanced touch | |
| 14 | * instructions. We use a stream ID of 0 for the load side and | |
| 15 | * 1 for the store side. Since source and destination are page | |
| 16 | * aligned we don't need to clear the bottom 7 bits of either | |
| 17 | * address. | |
| 18 | */ | |
| 280a5ba2 | 19 | ori r9,r3,1 /* stream=1 => to */ |
| fde69282 AB |
20 | |
| 21 | #ifdef CONFIG_PPC_64K_PAGES | |
| 280a5ba2 MN |
22 | lis r7,0x0E01 /* depth=7 |
| 23 | * units/cachelines=512 */ | |
| fde69282 AB |
24 | #else |
| 25 | lis r7,0x0E00 /* depth=7 */ | |
| 280a5ba2 | 26 | ori r7,r7,0x1000 /* units/cachelines=32 */ |
| fde69282 AB |
27 | #endif |
| 28 | ori r10,r7,1 /* stream=1 */ | |
| 29 | ||
| 30 | lis r8,0x8000 /* GO=1 */ | |
| 31 | clrldi r8,r8,32 | |
| 32 | ||
| 280a5ba2 | 33 | /* setup read stream 0 */ |
| 8a583c0a AS |
34 | dcbt 0,r4,0b01000 /* addr from */ |
| 35 | dcbt 0,r7,0b01010 /* length and depth from */ | |
| 280a5ba2 | 36 | /* setup write stream 1 */ |
| 8a583c0a AS |
37 | dcbtst 0,r9,0b01000 /* addr to */ |
| 38 | dcbtst 0,r10,0b01010 /* length and depth to */ | |
| fde69282 | 39 | eieio |
| 8a583c0a | 40 | dcbt 0,r8,0b01010 /* all streams GO */ |
| fde69282 AB |
41 | |
| 42 | #ifdef CONFIG_ALTIVEC | |
| 43 | mflr r0 | |
| 752a6422 UW |
44 | std r3,-STACKFRAMESIZE+STK_REG(R31)(r1) |
| 45 | std r4,-STACKFRAMESIZE+STK_REG(R30)(r1) | |
| fde69282 AB |
46 | std r0,16(r1) |
| 47 | stdu r1,-STACKFRAMESIZE(r1) | |
| 4e991e3c | 48 | bl CFUNC(enter_vmx_ops) |
| fde69282 AB |
49 | cmpwi r3,0 |
| 50 | ld r0,STACKFRAMESIZE+16(r1) | |
| 752a6422 UW |
51 | ld r3,STK_REG(R31)(r1) |
| 52 | ld r4,STK_REG(R30)(r1) | |
| fde69282 AB |
53 | mtlr r0 |
| 54 | ||
| 55 | li r0,(PAGE_SIZE/128) | |
| 56 | mtctr r0 | |
| 57 | ||
| 58 | beq .Lnonvmx_copy | |
| 59 | ||
| 60 | addi r1,r1,STACKFRAMESIZE | |
| 61 | ||
| 62 | li r6,16 | |
| 63 | li r7,32 | |
| 64 | li r8,48 | |
| 65 | li r9,64 | |
| 66 | li r10,80 | |
| 67 | li r11,96 | |
| 68 | li r12,112 | |
| 69 | ||
| 70 | .align 5 | |
| 8a583c0a | 71 | 1: lvx v7,0,r4 |
| c2ce6f9f AB |
72 | lvx v6,r4,r6 |
| 73 | lvx v5,r4,r7 | |
| 74 | lvx v4,r4,r8 | |
| 75 | lvx v3,r4,r9 | |
| 76 | lvx v2,r4,r10 | |
| 77 | lvx v1,r4,r11 | |
| 78 | lvx v0,r4,r12 | |
| fde69282 | 79 | addi r4,r4,128 |
| 8a583c0a | 80 | stvx v7,0,r3 |
| c2ce6f9f AB |
81 | stvx v6,r3,r6 |
| 82 | stvx v5,r3,r7 | |
| 83 | stvx v4,r3,r8 | |
| 84 | stvx v3,r3,r9 | |
| 85 | stvx v2,r3,r10 | |
| 86 | stvx v1,r3,r11 | |
| 87 | stvx v0,r3,r12 | |
| fde69282 AB |
88 | addi r3,r3,128 |
| 89 | bdnz 1b | |
| 90 | ||
| 4e991e3c | 91 | b CFUNC(exit_vmx_ops) /* tail call optimise */ |
| fde69282 AB |
92 | |
| 93 | #else | |
| 94 | li r0,(PAGE_SIZE/128) | |
| 95 | mtctr r0 | |
| 96 | ||
| 97 | stdu r1,-STACKFRAMESIZE(r1) | |
| 98 | #endif | |
| 99 | ||
| 100 | .Lnonvmx_copy: | |
| c75df6f9 MN |
101 | std r14,STK_REG(R14)(r1) |
| 102 | std r15,STK_REG(R15)(r1) | |
| 103 | std r16,STK_REG(R16)(r1) | |
| 104 | std r17,STK_REG(R17)(r1) | |
| 105 | std r18,STK_REG(R18)(r1) | |
| 106 | std r19,STK_REG(R19)(r1) | |
| 107 | std r20,STK_REG(R20)(r1) | |
| fde69282 AB |
108 | |
| 109 | 1: ld r0,0(r4) | |
| 110 | ld r5,8(r4) | |
| 111 | ld r6,16(r4) | |
| 112 | ld r7,24(r4) | |
| 113 | ld r8,32(r4) | |
| 114 | ld r9,40(r4) | |
| 115 | ld r10,48(r4) | |
| 116 | ld r11,56(r4) | |
| 117 | ld r12,64(r4) | |
| 118 | ld r14,72(r4) | |
| 119 | ld r15,80(r4) | |
| 120 | ld r16,88(r4) | |
| 121 | ld r17,96(r4) | |
| 122 | ld r18,104(r4) | |
| 123 | ld r19,112(r4) | |
| 124 | ld r20,120(r4) | |
| 125 | addi r4,r4,128 | |
| 126 | std r0,0(r3) | |
| 127 | std r5,8(r3) | |
| 128 | std r6,16(r3) | |
| 129 | std r7,24(r3) | |
| 130 | std r8,32(r3) | |
| 131 | std r9,40(r3) | |
| 132 | std r10,48(r3) | |
| 133 | std r11,56(r3) | |
| 134 | std r12,64(r3) | |
| 135 | std r14,72(r3) | |
| 136 | std r15,80(r3) | |
| 137 | std r16,88(r3) | |
| 138 | std r17,96(r3) | |
| 139 | std r18,104(r3) | |
| 140 | std r19,112(r3) | |
| 141 | std r20,120(r3) | |
| 142 | addi r3,r3,128 | |
| 143 | bdnz 1b | |
| 144 | ||
| c75df6f9 MN |
145 | ld r14,STK_REG(R14)(r1) |
| 146 | ld r15,STK_REG(R15)(r1) | |
| 147 | ld r16,STK_REG(R16)(r1) | |
| 148 | ld r17,STK_REG(R17)(r1) | |
| 149 | ld r18,STK_REG(R18)(r1) | |
| 150 | ld r19,STK_REG(R19)(r1) | |
| 151 | ld r20,STK_REG(R20)(r1) | |
| fde69282 AB |
152 | addi r1,r1,STACKFRAMESIZE |
| 153 | blr |