Commit | Line | Data |
---|---|---|
5d8544e2 | 1 | #include <linux/linkage.h> |
4d47ce15 | 2 | #include <asm-generic/export.h> |
5d8544e2 | 3 | #include <asm/asm.h> |
6dd10d91 | 4 | #include <asm/asm-extable.h> |
5d8544e2 | 5 | #include <asm/csr.h> |
c2a658d4 AC |
6 | #include <asm/hwcap.h> |
7 | #include <asm/alternative-macros.h> | |
5d8544e2 | 8 | |
5d8544e2 | 9 | .macro fixup op reg addr lbl |
abc71bf0 | 10 | 100: |
5d8544e2 | 11 | \op \reg, \addr |
6dd10d91 | 12 | _asm_extable 100b, \lbl |
5d8544e2 PD |
13 | .endm |
14 | ||
76329c69 | 15 | SYM_FUNC_START(__asm_copy_to_user) |
c2a658d4 AC |
16 | #ifdef CONFIG_RISCV_ISA_V |
17 | ALTERNATIVE("j fallback_scalar_usercopy", "nop", 0, RISCV_ISA_EXT_v, CONFIG_RISCV_ISA_V) | |
18 | REG_L t0, riscv_v_usercopy_threshold | |
19 | bltu a2, t0, fallback_scalar_usercopy | |
20 | tail enter_vector_usercopy | |
21 | #endif | |
22 | SYM_FUNC_START(fallback_scalar_usercopy) | |
5d8544e2 PD |
23 | |
24 | /* Enable access to user memory */ | |
25 | li t6, SR_SUM | |
a4c3733d | 26 | csrs CSR_STATUS, t6 |
5d8544e2 | 27 | |
4b05b993 AG |
28 | /* |
29 | * Save the terminal address which will be used to compute the number | |
30 | * of bytes copied in case of a fixup exception. | |
31 | */ | |
32 | add t5, a0, a2 | |
5d8544e2 | 33 | |
5d8544e2 | 34 | /* |
ca6eaaa2 AT |
35 | * Register allocation for code below: |
36 | * a0 - start of uncopied dst | |
37 | * a1 - start of uncopied src | |
38 | * a2 - size | |
39 | * t0 - end of uncopied dst | |
5d8544e2 | 40 | */ |
ca6eaaa2 | 41 | add t0, a0, a2 |
ca6eaaa2 AT |
42 | |
43 | /* | |
44 | * Use byte copy only if too small. | |
ea196c54 | 45 | * SZREG holds 4 for RV32 and 8 for RV64 |
ca6eaaa2 | 46 | */ |
6010d300 | 47 | li a3, 9*SZREG /* size must be larger than size in word_copy */ |
ca6eaaa2 AT |
48 | bltu a2, a3, .Lbyte_copy_tail |
49 | ||
50 | /* | |
ea196c54 | 51 | * Copy first bytes until dst is aligned to word boundary. |
ca6eaaa2 AT |
52 | * a0 - start of dst |
53 | * t1 - start of aligned dst | |
54 | */ | |
55 | addi t1, a0, SZREG-1 | |
56 | andi t1, t1, ~(SZREG-1) | |
57 | /* dst is already aligned, skip */ | |
ea196c54 | 58 | beq a0, t1, .Lskip_align_dst |
5d8544e2 | 59 | 1: |
ca6eaaa2 AT |
60 | /* a5 - one byte for copying data */ |
61 | fixup lb a5, 0(a1), 10f | |
62 | addi a1, a1, 1 /* src */ | |
63 | fixup sb a5, 0(a0), 10f | |
64 | addi a0, a0, 1 /* dst */ | |
65 | bltu a0, t1, 1b /* t1 - start of aligned dst */ | |
66 | ||
ea196c54 | 67 | .Lskip_align_dst: |
ca6eaaa2 AT |
68 | /* |
69 | * Now dst is aligned. | |
70 | * Use shift-copy if src is misaligned. | |
71 | * Use word-copy if both src and dst are aligned because | |
72 | * can not use shift-copy which do not require shifting | |
73 | */ | |
74 | /* a1 - start of src */ | |
75 | andi a3, a1, SZREG-1 | |
76 | bnez a3, .Lshift_copy | |
77 | ||
78 | .Lword_copy: | |
79 | /* | |
80 | * Both src and dst are aligned, unrolled word copy | |
81 | * | |
82 | * a0 - start of aligned dst | |
83 | * a1 - start of aligned src | |
ca6eaaa2 AT |
84 | * t0 - end of aligned dst |
85 | */ | |
6010d300 | 86 | addi t0, t0, -(8*SZREG) /* not to over run */ |
5d8544e2 | 87 | 2: |
ca6eaaa2 AT |
88 | fixup REG_L a4, 0(a1), 10f |
89 | fixup REG_L a5, SZREG(a1), 10f | |
90 | fixup REG_L a6, 2*SZREG(a1), 10f | |
91 | fixup REG_L a7, 3*SZREG(a1), 10f | |
92 | fixup REG_L t1, 4*SZREG(a1), 10f | |
93 | fixup REG_L t2, 5*SZREG(a1), 10f | |
94 | fixup REG_L t3, 6*SZREG(a1), 10f | |
95 | fixup REG_L t4, 7*SZREG(a1), 10f | |
96 | fixup REG_S a4, 0(a0), 10f | |
97 | fixup REG_S a5, SZREG(a0), 10f | |
98 | fixup REG_S a6, 2*SZREG(a0), 10f | |
99 | fixup REG_S a7, 3*SZREG(a0), 10f | |
100 | fixup REG_S t1, 4*SZREG(a0), 10f | |
101 | fixup REG_S t2, 5*SZREG(a0), 10f | |
102 | fixup REG_S t3, 6*SZREG(a0), 10f | |
103 | fixup REG_S t4, 7*SZREG(a0), 10f | |
104 | addi a0, a0, 8*SZREG | |
105 | addi a1, a1, 8*SZREG | |
106 | bltu a0, t0, 2b | |
107 | ||
6010d300 | 108 | addi t0, t0, 8*SZREG /* revert to original value */ |
ca6eaaa2 AT |
109 | j .Lbyte_copy_tail |
110 | ||
111 | .Lshift_copy: | |
112 | ||
113 | /* | |
114 | * Word copy with shifting. | |
115 | * For misaligned copy we still perform aligned word copy, but | |
116 | * we need to use the value fetched from the previous iteration and | |
117 | * do some shifts. | |
ea196c54 | 118 | * This is safe because reading is less than a word size. |
ca6eaaa2 AT |
119 | * |
120 | * a0 - start of aligned dst | |
121 | * a1 - start of src | |
122 | * a3 - a1 & mask:(SZREG-1) | |
123 | * t0 - end of uncopied dst | |
124 | * t1 - end of aligned dst | |
125 | */ | |
126 | /* calculating aligned word boundary for dst */ | |
127 | andi t1, t0, ~(SZREG-1) | |
ea196c54 | 128 | /* Converting unaligned src to aligned src */ |
ca6eaaa2 AT |
129 | andi a1, a1, ~(SZREG-1) |
130 | ||
131 | /* | |
132 | * Calculate shifts | |
133 | * t3 - prev shift | |
134 | * t4 - current shift | |
135 | */ | |
22b5f16f | 136 | slli t3, a3, 3 /* converting bytes in a3 to bits */ |
ca6eaaa2 AT |
137 | li a5, SZREG*8 |
138 | sub t4, a5, t3 | |
139 | ||
ea196c54 | 140 | /* Load the first word to combine with second word */ |
ca6eaaa2 | 141 | fixup REG_L a5, 0(a1), 10f |
5d8544e2 PD |
142 | |
143 | 3: | |
ca6eaaa2 AT |
144 | /* Main shifting copy |
145 | * | |
146 | * a0 - start of aligned dst | |
147 | * a1 - start of aligned src | |
148 | * t1 - end of aligned dst | |
149 | */ | |
150 | ||
151 | /* At least one iteration will be executed */ | |
152 | srl a4, a5, t3 | |
153 | fixup REG_L a5, SZREG(a1), 10f | |
154 | addi a1, a1, SZREG | |
155 | sll a2, a5, t4 | |
156 | or a2, a2, a4 | |
157 | fixup REG_S a2, 0(a0), 10f | |
158 | addi a0, a0, SZREG | |
159 | bltu a0, t1, 3b | |
160 | ||
161 | /* Revert src to original unaligned value */ | |
162 | add a1, a1, a3 | |
163 | ||
164 | .Lbyte_copy_tail: | |
165 | /* | |
166 | * Byte copy anything left. | |
167 | * | |
168 | * a0 - start of remaining dst | |
169 | * a1 - start of remaining src | |
170 | * t0 - end of remaining dst | |
171 | */ | |
ea196c54 | 172 | bgeu a0, t0, .Lout_copy_user /* check if end of copy */ |
ca6eaaa2 AT |
173 | 4: |
174 | fixup lb a5, 0(a1), 10f | |
175 | addi a1, a1, 1 /* src */ | |
176 | fixup sb a5, 0(a0), 10f | |
177 | addi a0, a0, 1 /* dst */ | |
178 | bltu a0, t0, 4b /* t0 - end of dst */ | |
179 | ||
ea196c54 | 180 | .Lout_copy_user: |
5d8544e2 | 181 | /* Disable access to user memory */ |
a4c3733d | 182 | csrc CSR_STATUS, t6 |
ca6eaaa2 | 183 | li a0, 0 |
5d8544e2 | 184 | ret |
9d504f9a JZ |
185 | |
186 | /* Exception fixup code */ | |
187 | 10: | |
188 | /* Disable access to user memory */ | |
c08b4848 | 189 | csrc CSR_STATUS, t6 |
4b05b993 | 190 | sub a0, t5, a0 |
9d504f9a | 191 | ret |
76329c69 | 192 | SYM_FUNC_END(__asm_copy_to_user) |
c2a658d4 | 193 | SYM_FUNC_END(fallback_scalar_usercopy) |
4d47ce15 | 194 | EXPORT_SYMBOL(__asm_copy_to_user) |
76329c69 | 195 | SYM_FUNC_ALIAS(__asm_copy_from_user, __asm_copy_to_user) |
4d47ce15 | 196 | EXPORT_SYMBOL(__asm_copy_from_user) |
5d8544e2 PD |
197 | |
198 | ||
76329c69 | 199 | SYM_FUNC_START(__clear_user) |
5d8544e2 PD |
200 | |
201 | /* Enable access to user memory */ | |
202 | li t6, SR_SUM | |
a4c3733d | 203 | csrs CSR_STATUS, t6 |
5d8544e2 PD |
204 | |
205 | add a3, a0, a1 | |
206 | addi t0, a0, SZREG-1 | |
207 | andi t1, a3, ~(SZREG-1) | |
208 | andi t0, t0, ~(SZREG-1) | |
209 | /* | |
210 | * a3: terminal address of target region | |
211 | * t0: lowest doubleword-aligned address in target region | |
212 | * t1: highest doubleword-aligned address in target region | |
213 | */ | |
214 | bgeu t0, t1, 2f | |
215 | bltu a0, t0, 4f | |
216 | 1: | |
ebcbd75e | 217 | fixup REG_S, zero, (a0), 11f |
5d8544e2 PD |
218 | addi a0, a0, SZREG |
219 | bltu a0, t1, 1b | |
220 | 2: | |
221 | bltu a0, a3, 5f | |
222 | ||
223 | 3: | |
224 | /* Disable access to user memory */ | |
a4c3733d | 225 | csrc CSR_STATUS, t6 |
5d8544e2 PD |
226 | li a0, 0 |
227 | ret | |
228 | 4: /* Edge case: unalignment */ | |
ebcbd75e | 229 | fixup sb, zero, (a0), 11f |
5d8544e2 PD |
230 | addi a0, a0, 1 |
231 | bltu a0, t0, 4b | |
232 | j 1b | |
233 | 5: /* Edge case: remainder */ | |
ebcbd75e | 234 | fixup sb, zero, (a0), 11f |
5d8544e2 PD |
235 | addi a0, a0, 1 |
236 | bltu a0, a3, 5b | |
237 | j 3b | |
5d8544e2 | 238 | |
9d504f9a | 239 | /* Exception fixup code */ |
ebcbd75e | 240 | 11: |
9d504f9a | 241 | /* Disable access to user memory */ |
c08b4848 | 242 | csrc CSR_STATUS, t6 |
4b05b993 | 243 | sub a0, a3, a0 |
5d8544e2 | 244 | ret |
76329c69 | 245 | SYM_FUNC_END(__clear_user) |
9d504f9a | 246 | EXPORT_SYMBOL(__clear_user) |