Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
373cd784 JH |
2 | /* |
3 | * User address space access functions. | |
4 | * The non-inlined parts of asm-metag/uaccess.h are here. | |
5 | * | |
6 | * Copyright (C) 2006, Imagination Technologies. | |
7 | * Copyright (C) 2000, Axis Communications AB. | |
8 | * | |
9 | * Written by Hans-Peter Nilsson. | |
10 | * Pieces used from memcpy, originally by Kenny Ranerup long time ago. | |
11 | * Modified for Meta by Will Newton. | |
12 | */ | |
13 | ||
9da3ee9a | 14 | #include <linux/export.h> |
373cd784 JH |
15 | #include <linux/uaccess.h> |
16 | #include <asm/cache.h> /* def of L1_CACHE_BYTES */ | |
17 | ||
18 | #define USE_RAPF | |
19 | #define RAPF_MIN_BUF_SIZE (3*L1_CACHE_BYTES) | |
20 | ||
21 | ||
22 | /* The "double write" in this code is because the Meta will not fault | |
23 | * immediately unless the memory pipe is forced to by e.g. a data stall or | |
24 | * another memory op. The second write should be discarded by the write | |
25 | * combiner so should have virtually no cost. | |
26 | */ | |
27 | ||
28 | #define __asm_copy_user_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
29 | asm volatile ( \ | |
30 | COPY \ | |
31 | "1:\n" \ | |
32 | " .section .fixup,\"ax\"\n" \ | |
373cd784 JH |
33 | FIXUP \ |
34 | " MOVT D1Ar1,#HI(1b)\n" \ | |
35 | " JUMP D1Ar1,#LO(1b)\n" \ | |
36 | " .previous\n" \ | |
37 | " .section __ex_table,\"a\"\n" \ | |
38 | TENTRY \ | |
39 | " .previous\n" \ | |
40 | : "=r" (to), "=r" (from), "=r" (ret) \ | |
41 | : "0" (to), "1" (from), "2" (ret) \ | |
42 | : "D1Ar1", "memory") | |
43 | ||
44 | ||
45 | #define __asm_copy_to_user_1(to, from, ret) \ | |
46 | __asm_copy_user_cont(to, from, ret, \ | |
47 | " GETB D1Ar1,[%1++]\n" \ | |
48 | " SETB [%0],D1Ar1\n" \ | |
49 | "2: SETB [%0++],D1Ar1\n", \ | |
50 | "3: ADD %2,%2,#1\n", \ | |
51 | " .long 2b,3b\n") | |
52 | ||
53 | #define __asm_copy_to_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
54 | __asm_copy_user_cont(to, from, ret, \ | |
55 | " GETW D1Ar1,[%1++]\n" \ | |
56 | " SETW [%0],D1Ar1\n" \ | |
57 | "2: SETW [%0++],D1Ar1\n" COPY, \ | |
58 | "3: ADD %2,%2,#2\n" FIXUP, \ | |
59 | " .long 2b,3b\n" TENTRY) | |
60 | ||
61 | #define __asm_copy_to_user_2(to, from, ret) \ | |
62 | __asm_copy_to_user_2x_cont(to, from, ret, "", "", "") | |
63 | ||
64 | #define __asm_copy_to_user_3(to, from, ret) \ | |
65 | __asm_copy_to_user_2x_cont(to, from, ret, \ | |
66 | " GETB D1Ar1,[%1++]\n" \ | |
67 | " SETB [%0],D1Ar1\n" \ | |
68 | "4: SETB [%0++],D1Ar1\n", \ | |
69 | "5: ADD %2,%2,#1\n", \ | |
70 | " .long 4b,5b\n") | |
71 | ||
72 | #define __asm_copy_to_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
73 | __asm_copy_user_cont(to, from, ret, \ | |
74 | " GETD D1Ar1,[%1++]\n" \ | |
75 | " SETD [%0],D1Ar1\n" \ | |
76 | "2: SETD [%0++],D1Ar1\n" COPY, \ | |
77 | "3: ADD %2,%2,#4\n" FIXUP, \ | |
78 | " .long 2b,3b\n" TENTRY) | |
79 | ||
80 | #define __asm_copy_to_user_4(to, from, ret) \ | |
81 | __asm_copy_to_user_4x_cont(to, from, ret, "", "", "") | |
82 | ||
83 | #define __asm_copy_to_user_5(to, from, ret) \ | |
84 | __asm_copy_to_user_4x_cont(to, from, ret, \ | |
85 | " GETB D1Ar1,[%1++]\n" \ | |
86 | " SETB [%0],D1Ar1\n" \ | |
87 | "4: SETB [%0++],D1Ar1\n", \ | |
88 | "5: ADD %2,%2,#1\n", \ | |
89 | " .long 4b,5b\n") | |
90 | ||
91 | #define __asm_copy_to_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
92 | __asm_copy_to_user_4x_cont(to, from, ret, \ | |
93 | " GETW D1Ar1,[%1++]\n" \ | |
94 | " SETW [%0],D1Ar1\n" \ | |
95 | "4: SETW [%0++],D1Ar1\n" COPY, \ | |
96 | "5: ADD %2,%2,#2\n" FIXUP, \ | |
97 | " .long 4b,5b\n" TENTRY) | |
98 | ||
99 | #define __asm_copy_to_user_6(to, from, ret) \ | |
100 | __asm_copy_to_user_6x_cont(to, from, ret, "", "", "") | |
101 | ||
102 | #define __asm_copy_to_user_7(to, from, ret) \ | |
103 | __asm_copy_to_user_6x_cont(to, from, ret, \ | |
104 | " GETB D1Ar1,[%1++]\n" \ | |
105 | " SETB [%0],D1Ar1\n" \ | |
106 | "6: SETB [%0++],D1Ar1\n", \ | |
107 | "7: ADD %2,%2,#1\n", \ | |
108 | " .long 6b,7b\n") | |
109 | ||
110 | #define __asm_copy_to_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
111 | __asm_copy_to_user_4x_cont(to, from, ret, \ | |
112 | " GETD D1Ar1,[%1++]\n" \ | |
113 | " SETD [%0],D1Ar1\n" \ | |
114 | "4: SETD [%0++],D1Ar1\n" COPY, \ | |
115 | "5: ADD %2,%2,#4\n" FIXUP, \ | |
116 | " .long 4b,5b\n" TENTRY) | |
117 | ||
118 | #define __asm_copy_to_user_8(to, from, ret) \ | |
119 | __asm_copy_to_user_8x_cont(to, from, ret, "", "", "") | |
120 | ||
121 | #define __asm_copy_to_user_9(to, from, ret) \ | |
122 | __asm_copy_to_user_8x_cont(to, from, ret, \ | |
123 | " GETB D1Ar1,[%1++]\n" \ | |
124 | " SETB [%0],D1Ar1\n" \ | |
125 | "6: SETB [%0++],D1Ar1\n", \ | |
126 | "7: ADD %2,%2,#1\n", \ | |
127 | " .long 6b,7b\n") | |
128 | ||
129 | #define __asm_copy_to_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
130 | __asm_copy_to_user_8x_cont(to, from, ret, \ | |
131 | " GETW D1Ar1,[%1++]\n" \ | |
132 | " SETW [%0],D1Ar1\n" \ | |
133 | "6: SETW [%0++],D1Ar1\n" COPY, \ | |
134 | "7: ADD %2,%2,#2\n" FIXUP, \ | |
135 | " .long 6b,7b\n" TENTRY) | |
136 | ||
137 | #define __asm_copy_to_user_10(to, from, ret) \ | |
138 | __asm_copy_to_user_10x_cont(to, from, ret, "", "", "") | |
139 | ||
140 | #define __asm_copy_to_user_11(to, from, ret) \ | |
141 | __asm_copy_to_user_10x_cont(to, from, ret, \ | |
142 | " GETB D1Ar1,[%1++]\n" \ | |
143 | " SETB [%0],D1Ar1\n" \ | |
144 | "8: SETB [%0++],D1Ar1\n", \ | |
145 | "9: ADD %2,%2,#1\n", \ | |
146 | " .long 8b,9b\n") | |
147 | ||
148 | #define __asm_copy_to_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
149 | __asm_copy_to_user_8x_cont(to, from, ret, \ | |
150 | " GETD D1Ar1,[%1++]\n" \ | |
151 | " SETD [%0],D1Ar1\n" \ | |
152 | "6: SETD [%0++],D1Ar1\n" COPY, \ | |
153 | "7: ADD %2,%2,#4\n" FIXUP, \ | |
154 | " .long 6b,7b\n" TENTRY) | |
155 | #define __asm_copy_to_user_12(to, from, ret) \ | |
156 | __asm_copy_to_user_12x_cont(to, from, ret, "", "", "") | |
157 | ||
158 | #define __asm_copy_to_user_13(to, from, ret) \ | |
159 | __asm_copy_to_user_12x_cont(to, from, ret, \ | |
160 | " GETB D1Ar1,[%1++]\n" \ | |
161 | " SETB [%0],D1Ar1\n" \ | |
162 | "8: SETB [%0++],D1Ar1\n", \ | |
163 | "9: ADD %2,%2,#1\n", \ | |
164 | " .long 8b,9b\n") | |
165 | ||
166 | #define __asm_copy_to_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
167 | __asm_copy_to_user_12x_cont(to, from, ret, \ | |
168 | " GETW D1Ar1,[%1++]\n" \ | |
169 | " SETW [%0],D1Ar1\n" \ | |
170 | "8: SETW [%0++],D1Ar1\n" COPY, \ | |
171 | "9: ADD %2,%2,#2\n" FIXUP, \ | |
172 | " .long 8b,9b\n" TENTRY) | |
173 | ||
174 | #define __asm_copy_to_user_14(to, from, ret) \ | |
175 | __asm_copy_to_user_14x_cont(to, from, ret, "", "", "") | |
176 | ||
177 | #define __asm_copy_to_user_15(to, from, ret) \ | |
178 | __asm_copy_to_user_14x_cont(to, from, ret, \ | |
179 | " GETB D1Ar1,[%1++]\n" \ | |
180 | " SETB [%0],D1Ar1\n" \ | |
181 | "10: SETB [%0++],D1Ar1\n", \ | |
182 | "11: ADD %2,%2,#1\n", \ | |
183 | " .long 10b,11b\n") | |
184 | ||
185 | #define __asm_copy_to_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
186 | __asm_copy_to_user_12x_cont(to, from, ret, \ | |
187 | " GETD D1Ar1,[%1++]\n" \ | |
188 | " SETD [%0],D1Ar1\n" \ | |
189 | "8: SETD [%0++],D1Ar1\n" COPY, \ | |
190 | "9: ADD %2,%2,#4\n" FIXUP, \ | |
191 | " .long 8b,9b\n" TENTRY) | |
192 | ||
193 | #define __asm_copy_to_user_16(to, from, ret) \ | |
194 | __asm_copy_to_user_16x_cont(to, from, ret, "", "", "") | |
195 | ||
196 | #define __asm_copy_to_user_8x64(to, from, ret) \ | |
197 | asm volatile ( \ | |
198 | " GETL D0Ar2,D1Ar1,[%1++]\n" \ | |
199 | " SETL [%0],D0Ar2,D1Ar1\n" \ | |
200 | "2: SETL [%0++],D0Ar2,D1Ar1\n" \ | |
201 | "1:\n" \ | |
202 | " .section .fixup,\"ax\"\n" \ | |
203 | "3: ADD %2,%2,#8\n" \ | |
204 | " MOVT D0Ar2,#HI(1b)\n" \ | |
205 | " JUMP D0Ar2,#LO(1b)\n" \ | |
206 | " .previous\n" \ | |
207 | " .section __ex_table,\"a\"\n" \ | |
208 | " .long 2b,3b\n" \ | |
209 | " .previous\n" \ | |
210 | : "=r" (to), "=r" (from), "=r" (ret) \ | |
211 | : "0" (to), "1" (from), "2" (ret) \ | |
212 | : "D1Ar1", "D0Ar2", "memory") | |
213 | ||
214 | /* | |
215 | * optimized copying loop using RAPF when 64 bit aligned | |
216 | * | |
217 | * n will be automatically decremented inside the loop | |
218 | * ret will be left intact. if error occurs we will rewind | |
219 | * so that the original non optimized code will fill up | |
220 | * this value correctly. | |
221 | * | |
222 | * on fault: | |
223 | * > n will hold total number of uncopied bytes | |
224 | * | |
225 | * > {'to','from'} will be rewind back so that | |
226 | * the non-optimized code will do the proper fix up | |
227 | * | |
228 | * DCACHE drops the cacheline which helps in reducing cache | |
229 | * pollution. | |
230 | * | |
231 | * We introduce an extra SETL at the end of the loop to | |
232 | * ensure we don't fall off the loop before we catch all | |
233 | * erros. | |
234 | * | |
235 | * NOTICE: | |
236 | * LSM_STEP in TXSTATUS must be cleared in fix up code. | |
237 | * since we're using M{S,G}ETL, a fault might happen at | |
238 | * any address in the middle of M{S,G}ETL causing | |
239 | * the value of LSM_STEP to be incorrect which can | |
240 | * cause subsequent use of M{S,G}ET{L,D} to go wrong. | |
241 | * ie: if LSM_STEP was 1 when a fault occurs, the | |
242 | * next call to M{S,G}ET{L,D} will skip the first | |
243 | * copy/getting as it think that the first 1 has already | |
244 | * been done. | |
245 | * | |
246 | */ | |
247 | #define __asm_copy_user_64bit_rapf_loop( \ | |
248 | to, from, ret, n, id, FIXUP) \ | |
249 | asm volatile ( \ | |
049520dc JH |
250 | ".balign 8\n" \ |
251 | " MOV RAPF, %1\n" \ | |
252 | " MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \ | |
253 | " MOV D0Ar6, #0\n" \ | |
254 | " LSR D1Ar5, %3, #6\n" \ | |
255 | " SUB TXRPT, D1Ar5, #2\n" \ | |
256 | " MOV RAPF, %1\n" \ | |
373cd784 | 257 | "$Lloop"id":\n" \ |
049520dc JH |
258 | " ADD RAPF, %1, #64\n" \ |
259 | "21: MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
260 | "22: MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
261 | "23: SUB %3, %3, #32\n" \ | |
262 | "24: MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
263 | "25: MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
264 | "26: SUB %3, %3, #32\n" \ | |
265 | " DCACHE [%1+#-64], D0Ar6\n" \ | |
266 | " BR $Lloop"id"\n" \ | |
373cd784 | 267 | \ |
049520dc JH |
268 | " MOV RAPF, %1\n" \ |
269 | "27: MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
270 | "28: MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
271 | "29: SUB %3, %3, #32\n" \ | |
272 | "30: MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
273 | "31: MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
fc1b759a | 274 | "32: SETL [%0+#-8], D0.7, D1.7\n" \ |
049520dc JH |
275 | " SUB %3, %3, #32\n" \ |
276 | "1: DCACHE [%1+#-64], D0Ar6\n" \ | |
277 | " GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \ | |
278 | " GETL D0FrT, D1RtP, [A0StP+#-32]\n" \ | |
279 | " GETL D0.5, D1.5, [A0StP+#-24]\n" \ | |
280 | " GETL D0.6, D1.6, [A0StP+#-16]\n" \ | |
281 | " GETL D0.7, D1.7, [A0StP+#-8]\n" \ | |
282 | " SUB A0StP, A0StP, #40\n" \ | |
373cd784 | 283 | " .section .fixup,\"ax\"\n" \ |
049520dc | 284 | "3: MOV D0Ar2, TXSTATUS\n" \ |
373cd784 JH |
285 | " MOV D1Ar1, TXSTATUS\n" \ |
286 | " AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \ | |
287 | " MOV TXSTATUS, D1Ar1\n" \ | |
288 | FIXUP \ | |
049520dc JH |
289 | " MOVT D0Ar2, #HI(1b)\n" \ |
290 | " JUMP D0Ar2, #LO(1b)\n" \ | |
373cd784 JH |
291 | " .previous\n" \ |
292 | " .section __ex_table,\"a\"\n" \ | |
293 | " .long 21b,3b\n" \ | |
294 | " .long 22b,3b\n" \ | |
295 | " .long 23b,3b\n" \ | |
296 | " .long 24b,3b\n" \ | |
297 | " .long 25b,3b\n" \ | |
298 | " .long 26b,3b\n" \ | |
299 | " .long 27b,3b\n" \ | |
300 | " .long 28b,3b\n" \ | |
b884a190 JH |
301 | " .long 29b,3b\n" \ |
302 | " .long 30b,3b\n" \ | |
303 | " .long 31b,3b\n" \ | |
304 | " .long 32b,3b\n" \ | |
373cd784 JH |
305 | " .previous\n" \ |
306 | : "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \ | |
307 | : "0" (to), "1" (from), "2" (ret), "3" (n) \ | |
fd40eee1 | 308 | : "D1Ar1", "D0Ar2", "cc", "memory") |
373cd784 JH |
309 | |
310 | /* rewind 'to' and 'from' pointers when a fault occurs | |
311 | * | |
312 | * Rationale: | |
313 | * A fault always occurs on writing to user buffer. A fault | |
314 | * is at a single address, so we need to rewind by only 4 | |
315 | * bytes. | |
316 | * Since we do a complete read from kernel buffer before | |
317 | * writing, we need to rewind it also. The amount to be | |
318 | * rewind equals the number of faulty writes in MSETD | |
319 | * which is: [4 - (LSM_STEP-1)]*8 | |
320 | * LSM_STEP is bits 10:8 in TXSTATUS which is already read | |
321 | * and stored in D0Ar2 | |
322 | * | |
323 | * NOTE: If a fault occurs at the last operation in M{G,S}ETL | |
324 | * LSM_STEP will be 0. ie: we do 4 writes in our case, if | |
325 | * a fault happens at the 4th write, LSM_STEP will be 0 | |
326 | * instead of 4. The code copes with that. | |
327 | * | |
328 | * n is updated by the number of successful writes, which is: | |
329 | * n = n - (LSM_STEP-1)*8 | |
330 | */ | |
331 | #define __asm_copy_to_user_64bit_rapf_loop(to, from, ret, n, id)\ | |
332 | __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \ | |
333 | "LSR D0Ar2, D0Ar2, #8\n" \ | |
fd40eee1 | 334 | "ANDS D0Ar2, D0Ar2, #0x7\n" \ |
373cd784 JH |
335 | "ADDZ D0Ar2, D0Ar2, #4\n" \ |
336 | "SUB D0Ar2, D0Ar2, #1\n" \ | |
337 | "MOV D1Ar1, #4\n" \ | |
338 | "SUB D0Ar2, D1Ar1, D0Ar2\n" \ | |
339 | "LSL D0Ar2, D0Ar2, #3\n" \ | |
340 | "LSL D1Ar1, D1Ar1, #3\n" \ | |
341 | "SUB D1Ar1, D1Ar1, D0Ar2\n" \ | |
342 | "SUB %0, %0, #8\n" \ | |
343 | "SUB %1, %1,D0Ar2\n" \ | |
344 | "SUB %3, %3, D1Ar1\n") | |
345 | ||
346 | /* | |
347 | * optimized copying loop using RAPF when 32 bit aligned | |
348 | * | |
349 | * n will be automatically decremented inside the loop | |
350 | * ret will be left intact. if error occurs we will rewind | |
351 | * so that the original non optimized code will fill up | |
352 | * this value correctly. | |
353 | * | |
354 | * on fault: | |
355 | * > n will hold total number of uncopied bytes | |
356 | * | |
357 | * > {'to','from'} will be rewind back so that | |
358 | * the non-optimized code will do the proper fix up | |
359 | * | |
360 | * DCACHE drops the cacheline which helps in reducing cache | |
361 | * pollution. | |
362 | * | |
363 | * We introduce an extra SETD at the end of the loop to | |
364 | * ensure we don't fall off the loop before we catch all | |
365 | * erros. | |
366 | * | |
367 | * NOTICE: | |
368 | * LSM_STEP in TXSTATUS must be cleared in fix up code. | |
369 | * since we're using M{S,G}ETL, a fault might happen at | |
370 | * any address in the middle of M{S,G}ETL causing | |
371 | * the value of LSM_STEP to be incorrect which can | |
372 | * cause subsequent use of M{S,G}ET{L,D} to go wrong. | |
373 | * ie: if LSM_STEP was 1 when a fault occurs, the | |
374 | * next call to M{S,G}ET{L,D} will skip the first | |
375 | * copy/getting as it think that the first 1 has already | |
376 | * been done. | |
377 | * | |
378 | */ | |
379 | #define __asm_copy_user_32bit_rapf_loop( \ | |
380 | to, from, ret, n, id, FIXUP) \ | |
381 | asm volatile ( \ | |
049520dc JH |
382 | ".balign 8\n" \ |
383 | " MOV RAPF, %1\n" \ | |
384 | " MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \ | |
385 | " MOV D0Ar6, #0\n" \ | |
386 | " LSR D1Ar5, %3, #6\n" \ | |
387 | " SUB TXRPT, D1Ar5, #2\n" \ | |
388 | " MOV RAPF, %1\n" \ | |
389 | "$Lloop"id":\n" \ | |
390 | " ADD RAPF, %1, #64\n" \ | |
391 | "21: MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
392 | "22: MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
393 | "23: SUB %3, %3, #16\n" \ | |
394 | "24: MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
395 | "25: MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
396 | "26: SUB %3, %3, #16\n" \ | |
397 | "27: MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
398 | "28: MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
399 | "29: SUB %3, %3, #16\n" \ | |
400 | "30: MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
401 | "31: MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
402 | "32: SUB %3, %3, #16\n" \ | |
403 | " DCACHE [%1+#-64], D0Ar6\n" \ | |
404 | " BR $Lloop"id"\n" \ | |
373cd784 | 405 | \ |
049520dc JH |
406 | " MOV RAPF, %1\n" \ |
407 | "33: MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
408 | "34: MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
409 | "35: SUB %3, %3, #16\n" \ | |
410 | "36: MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
411 | "37: MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
412 | "38: SUB %3, %3, #16\n" \ | |
413 | "39: MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
414 | "40: MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
415 | "41: SUB %3, %3, #16\n" \ | |
416 | "42: MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
417 | "43: MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
fc1b759a | 418 | "44: SETD [%0+#-4], D0.7\n" \ |
049520dc JH |
419 | " SUB %3, %3, #16\n" \ |
420 | "1: DCACHE [%1+#-64], D0Ar6\n" \ | |
421 | " GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \ | |
422 | " GETL D0FrT, D1RtP, [A0StP+#-32]\n" \ | |
423 | " GETL D0.5, D1.5, [A0StP+#-24]\n" \ | |
424 | " GETL D0.6, D1.6, [A0StP+#-16]\n" \ | |
425 | " GETL D0.7, D1.7, [A0StP+#-8]\n" \ | |
426 | " SUB A0StP, A0StP, #40\n" \ | |
373cd784 | 427 | " .section .fixup,\"ax\"\n" \ |
049520dc | 428 | "3: MOV D0Ar2, TXSTATUS\n" \ |
373cd784 JH |
429 | " MOV D1Ar1, TXSTATUS\n" \ |
430 | " AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \ | |
431 | " MOV TXSTATUS, D1Ar1\n" \ | |
432 | FIXUP \ | |
049520dc JH |
433 | " MOVT D0Ar2, #HI(1b)\n" \ |
434 | " JUMP D0Ar2, #LO(1b)\n" \ | |
373cd784 JH |
435 | " .previous\n" \ |
436 | " .section __ex_table,\"a\"\n" \ | |
437 | " .long 21b,3b\n" \ | |
438 | " .long 22b,3b\n" \ | |
439 | " .long 23b,3b\n" \ | |
440 | " .long 24b,3b\n" \ | |
441 | " .long 25b,3b\n" \ | |
442 | " .long 26b,3b\n" \ | |
443 | " .long 27b,3b\n" \ | |
444 | " .long 28b,3b\n" \ | |
445 | " .long 29b,3b\n" \ | |
446 | " .long 30b,3b\n" \ | |
447 | " .long 31b,3b\n" \ | |
448 | " .long 32b,3b\n" \ | |
449 | " .long 33b,3b\n" \ | |
450 | " .long 34b,3b\n" \ | |
451 | " .long 35b,3b\n" \ | |
452 | " .long 36b,3b\n" \ | |
b884a190 JH |
453 | " .long 37b,3b\n" \ |
454 | " .long 38b,3b\n" \ | |
455 | " .long 39b,3b\n" \ | |
456 | " .long 40b,3b\n" \ | |
457 | " .long 41b,3b\n" \ | |
458 | " .long 42b,3b\n" \ | |
459 | " .long 43b,3b\n" \ | |
460 | " .long 44b,3b\n" \ | |
373cd784 JH |
461 | " .previous\n" \ |
462 | : "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \ | |
463 | : "0" (to), "1" (from), "2" (ret), "3" (n) \ | |
fd40eee1 | 464 | : "D1Ar1", "D0Ar2", "cc", "memory") |
373cd784 JH |
465 | |
466 | /* rewind 'to' and 'from' pointers when a fault occurs | |
467 | * | |
468 | * Rationale: | |
469 | * A fault always occurs on writing to user buffer. A fault | |
470 | * is at a single address, so we need to rewind by only 4 | |
471 | * bytes. | |
472 | * Since we do a complete read from kernel buffer before | |
473 | * writing, we need to rewind it also. The amount to be | |
474 | * rewind equals the number of faulty writes in MSETD | |
475 | * which is: [4 - (LSM_STEP-1)]*4 | |
476 | * LSM_STEP is bits 10:8 in TXSTATUS which is already read | |
477 | * and stored in D0Ar2 | |
478 | * | |
479 | * NOTE: If a fault occurs at the last operation in M{G,S}ETL | |
480 | * LSM_STEP will be 0. ie: we do 4 writes in our case, if | |
481 | * a fault happens at the 4th write, LSM_STEP will be 0 | |
482 | * instead of 4. The code copes with that. | |
483 | * | |
484 | * n is updated by the number of successful writes, which is: | |
485 | * n = n - (LSM_STEP-1)*4 | |
486 | */ | |
487 | #define __asm_copy_to_user_32bit_rapf_loop(to, from, ret, n, id)\ | |
488 | __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \ | |
489 | "LSR D0Ar2, D0Ar2, #8\n" \ | |
fd40eee1 | 490 | "ANDS D0Ar2, D0Ar2, #0x7\n" \ |
373cd784 JH |
491 | "ADDZ D0Ar2, D0Ar2, #4\n" \ |
492 | "SUB D0Ar2, D0Ar2, #1\n" \ | |
493 | "MOV D1Ar1, #4\n" \ | |
494 | "SUB D0Ar2, D1Ar1, D0Ar2\n" \ | |
495 | "LSL D0Ar2, D0Ar2, #2\n" \ | |
496 | "LSL D1Ar1, D1Ar1, #2\n" \ | |
497 | "SUB D1Ar1, D1Ar1, D0Ar2\n" \ | |
498 | "SUB %0, %0, #4\n" \ | |
499 | "SUB %1, %1, D0Ar2\n" \ | |
500 | "SUB %3, %3, D1Ar1\n") | |
501 | ||
840db3f9 JH |
502 | unsigned long raw_copy_to_user(void __user *pdst, const void *psrc, |
503 | unsigned long n) | |
373cd784 JH |
504 | { |
505 | register char __user *dst asm ("A0.2") = pdst; | |
506 | register const char *src asm ("A1.2") = psrc; | |
507 | unsigned long retn = 0; | |
508 | ||
509 | if (n == 0) | |
510 | return 0; | |
511 | ||
512 | if ((unsigned long) src & 1) { | |
513 | __asm_copy_to_user_1(dst, src, retn); | |
514 | n--; | |
fb8ea062 JH |
515 | if (retn) |
516 | return retn + n; | |
373cd784 JH |
517 | } |
518 | if ((unsigned long) dst & 1) { | |
519 | /* Worst case - byte copy */ | |
520 | while (n > 0) { | |
521 | __asm_copy_to_user_1(dst, src, retn); | |
522 | n--; | |
fb8ea062 JH |
523 | if (retn) |
524 | return retn + n; | |
373cd784 JH |
525 | } |
526 | } | |
527 | if (((unsigned long) src & 2) && n >= 2) { | |
528 | __asm_copy_to_user_2(dst, src, retn); | |
529 | n -= 2; | |
fb8ea062 JH |
530 | if (retn) |
531 | return retn + n; | |
373cd784 JH |
532 | } |
533 | if ((unsigned long) dst & 2) { | |
534 | /* Second worst case - word copy */ | |
535 | while (n >= 2) { | |
536 | __asm_copy_to_user_2(dst, src, retn); | |
537 | n -= 2; | |
fb8ea062 JH |
538 | if (retn) |
539 | return retn + n; | |
373cd784 JH |
540 | } |
541 | } | |
542 | ||
543 | #ifdef USE_RAPF | |
544 | /* 64 bit copy loop */ | |
545 | if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) { | |
546 | if (n >= RAPF_MIN_BUF_SIZE) { | |
547 | /* copy user using 64 bit rapf copy */ | |
548 | __asm_copy_to_user_64bit_rapf_loop(dst, src, retn, | |
549 | n, "64cu"); | |
550 | } | |
551 | while (n >= 8) { | |
552 | __asm_copy_to_user_8x64(dst, src, retn); | |
553 | n -= 8; | |
fb8ea062 JH |
554 | if (retn) |
555 | return retn + n; | |
373cd784 JH |
556 | } |
557 | } | |
558 | if (n >= RAPF_MIN_BUF_SIZE) { | |
559 | /* copy user using 32 bit rapf copy */ | |
560 | __asm_copy_to_user_32bit_rapf_loop(dst, src, retn, n, "32cu"); | |
561 | } | |
562 | #else | |
563 | /* 64 bit copy loop */ | |
564 | if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) { | |
565 | while (n >= 8) { | |
566 | __asm_copy_to_user_8x64(dst, src, retn); | |
567 | n -= 8; | |
fb8ea062 JH |
568 | if (retn) |
569 | return retn + n; | |
373cd784 JH |
570 | } |
571 | } | |
572 | #endif | |
573 | ||
574 | while (n >= 16) { | |
575 | __asm_copy_to_user_16(dst, src, retn); | |
576 | n -= 16; | |
fb8ea062 JH |
577 | if (retn) |
578 | return retn + n; | |
373cd784 JH |
579 | } |
580 | ||
581 | while (n >= 4) { | |
582 | __asm_copy_to_user_4(dst, src, retn); | |
583 | n -= 4; | |
fb8ea062 JH |
584 | if (retn) |
585 | return retn + n; | |
373cd784 JH |
586 | } |
587 | ||
588 | switch (n) { | |
589 | case 0: | |
590 | break; | |
591 | case 1: | |
592 | __asm_copy_to_user_1(dst, src, retn); | |
593 | break; | |
594 | case 2: | |
595 | __asm_copy_to_user_2(dst, src, retn); | |
596 | break; | |
597 | case 3: | |
598 | __asm_copy_to_user_3(dst, src, retn); | |
599 | break; | |
600 | } | |
601 | ||
fb8ea062 JH |
602 | /* |
603 | * If we get here, retn correctly reflects the number of failing | |
604 | * bytes. | |
605 | */ | |
373cd784 JH |
606 | return retn; |
607 | } | |
840db3f9 | 608 | EXPORT_SYMBOL(raw_copy_to_user); |
373cd784 JH |
609 | |
610 | #define __asm_copy_from_user_1(to, from, ret) \ | |
611 | __asm_copy_user_cont(to, from, ret, \ | |
612 | " GETB D1Ar1,[%1++]\n" \ | |
613 | "2: SETB [%0++],D1Ar1\n", \ | |
563ddc10 | 614 | "3: ADD %2,%2,#1\n", \ |
373cd784 JH |
615 | " .long 2b,3b\n") |
616 | ||
617 | #define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
618 | __asm_copy_user_cont(to, from, ret, \ | |
619 | " GETW D1Ar1,[%1++]\n" \ | |
620 | "2: SETW [%0++],D1Ar1\n" COPY, \ | |
563ddc10 | 621 | "3: ADD %2,%2,#2\n" FIXUP, \ |
373cd784 JH |
622 | " .long 2b,3b\n" TENTRY) |
623 | ||
624 | #define __asm_copy_from_user_2(to, from, ret) \ | |
625 | __asm_copy_from_user_2x_cont(to, from, ret, "", "", "") | |
626 | ||
627 | #define __asm_copy_from_user_3(to, from, ret) \ | |
628 | __asm_copy_from_user_2x_cont(to, from, ret, \ | |
629 | " GETB D1Ar1,[%1++]\n" \ | |
630 | "4: SETB [%0++],D1Ar1\n", \ | |
563ddc10 | 631 | "5: ADD %2,%2,#1\n", \ |
373cd784 JH |
632 | " .long 4b,5b\n") |
633 | ||
634 | #define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
635 | __asm_copy_user_cont(to, from, ret, \ | |
636 | " GETD D1Ar1,[%1++]\n" \ | |
637 | "2: SETD [%0++],D1Ar1\n" COPY, \ | |
563ddc10 | 638 | "3: ADD %2,%2,#4\n" FIXUP, \ |
373cd784 JH |
639 | " .long 2b,3b\n" TENTRY) |
640 | ||
641 | #define __asm_copy_from_user_4(to, from, ret) \ | |
642 | __asm_copy_from_user_4x_cont(to, from, ret, "", "", "") | |
643 | ||
373cd784 JH |
644 | #define __asm_copy_from_user_8x64(to, from, ret) \ |
645 | asm volatile ( \ | |
646 | " GETL D0Ar2,D1Ar1,[%1++]\n" \ | |
647 | "2: SETL [%0++],D0Ar2,D1Ar1\n" \ | |
648 | "1:\n" \ | |
649 | " .section .fixup,\"ax\"\n" \ | |
373cd784 | 650 | "3: ADD %2,%2,#8\n" \ |
373cd784 JH |
651 | " MOVT D0Ar2,#HI(1b)\n" \ |
652 | " JUMP D0Ar2,#LO(1b)\n" \ | |
653 | " .previous\n" \ | |
654 | " .section __ex_table,\"a\"\n" \ | |
655 | " .long 2b,3b\n" \ | |
656 | " .previous\n" \ | |
657 | : "=a" (to), "=r" (from), "=r" (ret) \ | |
658 | : "0" (to), "1" (from), "2" (ret) \ | |
659 | : "D1Ar1", "D0Ar2", "memory") | |
660 | ||
661 | /* rewind 'from' pointer when a fault occurs | |
662 | * | |
663 | * Rationale: | |
664 | * A fault occurs while reading from user buffer, which is the | |
2c0b1df8 | 665 | * source. |
373cd784 JH |
666 | * Since we don't write to kernel buffer until we read first, |
667 | * the kernel buffer is at the right state and needn't be | |
2c0b1df8 JH |
668 | * corrected, but the source must be rewound to the beginning of |
669 | * the block, which is LSM_STEP*8 bytes. | |
670 | * LSM_STEP is bits 10:8 in TXSTATUS which is already read | |
671 | * and stored in D0Ar2 | |
672 | * | |
673 | * NOTE: If a fault occurs at the last operation in M{G,S}ETL | |
674 | * LSM_STEP will be 0. ie: we do 4 writes in our case, if | |
675 | * a fault happens at the 4th write, LSM_STEP will be 0 | |
676 | * instead of 4. The code copes with that. | |
373cd784 JH |
677 | */ |
678 | #define __asm_copy_from_user_64bit_rapf_loop(to, from, ret, n, id) \ | |
679 | __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \ | |
2c0b1df8 JH |
680 | "LSR D0Ar2, D0Ar2, #5\n" \ |
681 | "ANDS D0Ar2, D0Ar2, #0x38\n" \ | |
682 | "ADDZ D0Ar2, D0Ar2, #32\n" \ | |
683 | "SUB %1, %1, D0Ar2\n") | |
373cd784 JH |
684 | |
685 | /* rewind 'from' pointer when a fault occurs | |
686 | * | |
687 | * Rationale: | |
688 | * A fault occurs while reading from user buffer, which is the | |
2c0b1df8 | 689 | * source. |
373cd784 JH |
690 | * Since we don't write to kernel buffer until we read first, |
691 | * the kernel buffer is at the right state and needn't be | |
2c0b1df8 JH |
692 | * corrected, but the source must be rewound to the beginning of |
693 | * the block, which is LSM_STEP*4 bytes. | |
694 | * LSM_STEP is bits 10:8 in TXSTATUS which is already read | |
695 | * and stored in D0Ar2 | |
696 | * | |
697 | * NOTE: If a fault occurs at the last operation in M{G,S}ETL | |
698 | * LSM_STEP will be 0. ie: we do 4 writes in our case, if | |
699 | * a fault happens at the 4th write, LSM_STEP will be 0 | |
700 | * instead of 4. The code copes with that. | |
373cd784 JH |
701 | */ |
702 | #define __asm_copy_from_user_32bit_rapf_loop(to, from, ret, n, id) \ | |
703 | __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \ | |
2c0b1df8 JH |
704 | "LSR D0Ar2, D0Ar2, #6\n" \ |
705 | "ANDS D0Ar2, D0Ar2, #0x1c\n" \ | |
706 | "ADDZ D0Ar2, D0Ar2, #16\n" \ | |
707 | "SUB %1, %1, D0Ar2\n") | |
373cd784 JH |
708 | |
709 | ||
563ddc10 JH |
710 | /* |
711 | * Copy from user to kernel. The return-value is the number of bytes that were | |
712 | * inaccessible. | |
713 | */ | |
714 | unsigned long raw_copy_from_user(void *pdst, const void __user *psrc, | |
715 | unsigned long n) | |
373cd784 JH |
716 | { |
717 | register char *dst asm ("A0.2") = pdst; | |
718 | register const char __user *src asm ("A1.2") = psrc; | |
719 | unsigned long retn = 0; | |
720 | ||
721 | if (n == 0) | |
722 | return 0; | |
723 | ||
724 | if ((unsigned long) src & 1) { | |
725 | __asm_copy_from_user_1(dst, src, retn); | |
726 | n--; | |
22572119 | 727 | if (retn) |
563ddc10 | 728 | return retn + n; |
373cd784 JH |
729 | } |
730 | if ((unsigned long) dst & 1) { | |
731 | /* Worst case - byte copy */ | |
732 | while (n > 0) { | |
733 | __asm_copy_from_user_1(dst, src, retn); | |
734 | n--; | |
735 | if (retn) | |
563ddc10 | 736 | return retn + n; |
373cd784 JH |
737 | } |
738 | } | |
739 | if (((unsigned long) src & 2) && n >= 2) { | |
740 | __asm_copy_from_user_2(dst, src, retn); | |
741 | n -= 2; | |
22572119 | 742 | if (retn) |
563ddc10 | 743 | return retn + n; |
373cd784 JH |
744 | } |
745 | if ((unsigned long) dst & 2) { | |
746 | /* Second worst case - word copy */ | |
747 | while (n >= 2) { | |
748 | __asm_copy_from_user_2(dst, src, retn); | |
749 | n -= 2; | |
750 | if (retn) | |
563ddc10 | 751 | return retn + n; |
373cd784 JH |
752 | } |
753 | } | |
754 | ||
373cd784 JH |
755 | #ifdef USE_RAPF |
756 | /* 64 bit copy loop */ | |
757 | if (!(((unsigned long) src | (unsigned long) dst) & 7)) { | |
758 | if (n >= RAPF_MIN_BUF_SIZE) { | |
759 | /* Copy using fast 64bit rapf */ | |
760 | __asm_copy_from_user_64bit_rapf_loop(dst, src, retn, | |
761 | n, "64cuz"); | |
762 | } | |
763 | while (n >= 8) { | |
764 | __asm_copy_from_user_8x64(dst, src, retn); | |
765 | n -= 8; | |
766 | if (retn) | |
563ddc10 | 767 | return retn + n; |
373cd784 JH |
768 | } |
769 | } | |
770 | ||
771 | if (n >= RAPF_MIN_BUF_SIZE) { | |
772 | /* Copy using fast 32bit rapf */ | |
773 | __asm_copy_from_user_32bit_rapf_loop(dst, src, retn, | |
774 | n, "32cuz"); | |
775 | } | |
776 | #else | |
777 | /* 64 bit copy loop */ | |
778 | if (!(((unsigned long) src | (unsigned long) dst) & 7)) { | |
779 | while (n >= 8) { | |
780 | __asm_copy_from_user_8x64(dst, src, retn); | |
781 | n -= 8; | |
782 | if (retn) | |
563ddc10 | 783 | return retn + n; |
373cd784 JH |
784 | } |
785 | } | |
786 | #endif | |
787 | ||
788 | while (n >= 4) { | |
789 | __asm_copy_from_user_4(dst, src, retn); | |
790 | n -= 4; | |
791 | ||
792 | if (retn) | |
563ddc10 | 793 | return retn + n; |
373cd784 JH |
794 | } |
795 | ||
796 | /* If we get here, there were no memory read faults. */ | |
797 | switch (n) { | |
798 | /* These copies are at least "naturally aligned" (so we don't | |
799 | have to check each byte), due to the src alignment code. | |
800 | The *_3 case *will* get the correct count for retn. */ | |
801 | case 0: | |
802 | /* This case deliberately left in (if you have doubts check the | |
803 | generated assembly code). */ | |
804 | break; | |
805 | case 1: | |
806 | __asm_copy_from_user_1(dst, src, retn); | |
807 | break; | |
808 | case 2: | |
809 | __asm_copy_from_user_2(dst, src, retn); | |
810 | break; | |
811 | case 3: | |
812 | __asm_copy_from_user_3(dst, src, retn); | |
813 | break; | |
814 | } | |
815 | ||
816 | /* If we get here, retn correctly reflects the number of failing | |
817 | bytes. */ | |
818 | return retn; | |
373cd784 | 819 | } |
563ddc10 | 820 | EXPORT_SYMBOL(raw_copy_from_user); |
373cd784 JH |
821 | |
822 | #define __asm_clear_8x64(to, ret) \ | |
823 | asm volatile ( \ | |
824 | " MOV D0Ar2,#0\n" \ | |
825 | " MOV D1Ar1,#0\n" \ | |
826 | " SETL [%0],D0Ar2,D1Ar1\n" \ | |
827 | "2: SETL [%0++],D0Ar2,D1Ar1\n" \ | |
828 | "1:\n" \ | |
829 | " .section .fixup,\"ax\"\n" \ | |
830 | "3: ADD %1,%1,#8\n" \ | |
831 | " MOVT D0Ar2,#HI(1b)\n" \ | |
832 | " JUMP D0Ar2,#LO(1b)\n" \ | |
833 | " .previous\n" \ | |
834 | " .section __ex_table,\"a\"\n" \ | |
835 | " .long 2b,3b\n" \ | |
836 | " .previous\n" \ | |
837 | : "=r" (to), "=r" (ret) \ | |
838 | : "0" (to), "1" (ret) \ | |
839 | : "D1Ar1", "D0Ar2", "memory") | |
840 | ||
841 | /* Zero userspace. */ | |
842 | ||
843 | #define __asm_clear(to, ret, CLEAR, FIXUP, TENTRY) \ | |
844 | asm volatile ( \ | |
845 | " MOV D1Ar1,#0\n" \ | |
846 | CLEAR \ | |
847 | "1:\n" \ | |
848 | " .section .fixup,\"ax\"\n" \ | |
849 | FIXUP \ | |
850 | " MOVT D1Ar1,#HI(1b)\n" \ | |
851 | " JUMP D1Ar1,#LO(1b)\n" \ | |
852 | " .previous\n" \ | |
853 | " .section __ex_table,\"a\"\n" \ | |
854 | TENTRY \ | |
855 | " .previous" \ | |
856 | : "=r" (to), "=r" (ret) \ | |
857 | : "0" (to), "1" (ret) \ | |
858 | : "D1Ar1", "memory") | |
859 | ||
860 | #define __asm_clear_1(to, ret) \ | |
861 | __asm_clear(to, ret, \ | |
862 | " SETB [%0],D1Ar1\n" \ | |
863 | "2: SETB [%0++],D1Ar1\n", \ | |
864 | "3: ADD %1,%1,#1\n", \ | |
865 | " .long 2b,3b\n") | |
866 | ||
867 | #define __asm_clear_2(to, ret) \ | |
868 | __asm_clear(to, ret, \ | |
869 | " SETW [%0],D1Ar1\n" \ | |
870 | "2: SETW [%0++],D1Ar1\n", \ | |
871 | "3: ADD %1,%1,#2\n", \ | |
872 | " .long 2b,3b\n") | |
873 | ||
874 | #define __asm_clear_3(to, ret) \ | |
875 | __asm_clear(to, ret, \ | |
876 | "2: SETW [%0++],D1Ar1\n" \ | |
877 | " SETB [%0],D1Ar1\n" \ | |
878 | "3: SETB [%0++],D1Ar1\n", \ | |
879 | "4: ADD %1,%1,#2\n" \ | |
880 | "5: ADD %1,%1,#1\n", \ | |
881 | " .long 2b,4b\n" \ | |
882 | " .long 3b,5b\n") | |
883 | ||
884 | #define __asm_clear_4x_cont(to, ret, CLEAR, FIXUP, TENTRY) \ | |
885 | __asm_clear(to, ret, \ | |
886 | " SETD [%0],D1Ar1\n" \ | |
887 | "2: SETD [%0++],D1Ar1\n" CLEAR, \ | |
888 | "3: ADD %1,%1,#4\n" FIXUP, \ | |
889 | " .long 2b,3b\n" TENTRY) | |
890 | ||
891 | #define __asm_clear_4(to, ret) \ | |
892 | __asm_clear_4x_cont(to, ret, "", "", "") | |
893 | ||
894 | #define __asm_clear_8x_cont(to, ret, CLEAR, FIXUP, TENTRY) \ | |
895 | __asm_clear_4x_cont(to, ret, \ | |
896 | " SETD [%0],D1Ar1\n" \ | |
897 | "4: SETD [%0++],D1Ar1\n" CLEAR, \ | |
898 | "5: ADD %1,%1,#4\n" FIXUP, \ | |
899 | " .long 4b,5b\n" TENTRY) | |
900 | ||
901 | #define __asm_clear_8(to, ret) \ | |
902 | __asm_clear_8x_cont(to, ret, "", "", "") | |
903 | ||
904 | #define __asm_clear_12x_cont(to, ret, CLEAR, FIXUP, TENTRY) \ | |
905 | __asm_clear_8x_cont(to, ret, \ | |
906 | " SETD [%0],D1Ar1\n" \ | |
907 | "6: SETD [%0++],D1Ar1\n" CLEAR, \ | |
908 | "7: ADD %1,%1,#4\n" FIXUP, \ | |
909 | " .long 6b,7b\n" TENTRY) | |
910 | ||
911 | #define __asm_clear_12(to, ret) \ | |
912 | __asm_clear_12x_cont(to, ret, "", "", "") | |
913 | ||
914 | #define __asm_clear_16x_cont(to, ret, CLEAR, FIXUP, TENTRY) \ | |
915 | __asm_clear_12x_cont(to, ret, \ | |
916 | " SETD [%0],D1Ar1\n" \ | |
917 | "8: SETD [%0++],D1Ar1\n" CLEAR, \ | |
918 | "9: ADD %1,%1,#4\n" FIXUP, \ | |
919 | " .long 8b,9b\n" TENTRY) | |
920 | ||
921 | #define __asm_clear_16(to, ret) \ | |
922 | __asm_clear_16x_cont(to, ret, "", "", "") | |
923 | ||
924 | unsigned long __do_clear_user(void __user *pto, unsigned long pn) | |
925 | { | |
926 | register char __user *dst asm ("D0Re0") = pto; | |
927 | register unsigned long n asm ("D1Re0") = pn; | |
928 | register unsigned long retn asm ("D0Ar6") = 0; | |
929 | ||
930 | if ((unsigned long) dst & 1) { | |
931 | __asm_clear_1(dst, retn); | |
932 | n--; | |
933 | } | |
934 | ||
935 | if ((unsigned long) dst & 2) { | |
936 | __asm_clear_2(dst, retn); | |
937 | n -= 2; | |
938 | } | |
939 | ||
940 | /* 64 bit copy loop */ | |
941 | if (!((__force unsigned long) dst & 7)) { | |
942 | while (n >= 8) { | |
943 | __asm_clear_8x64(dst, retn); | |
944 | n -= 8; | |
945 | } | |
946 | } | |
947 | ||
948 | while (n >= 16) { | |
949 | __asm_clear_16(dst, retn); | |
950 | n -= 16; | |
951 | } | |
952 | ||
953 | while (n >= 4) { | |
954 | __asm_clear_4(dst, retn); | |
955 | n -= 4; | |
956 | } | |
957 | ||
958 | switch (n) { | |
959 | case 0: | |
960 | break; | |
961 | case 1: | |
962 | __asm_clear_1(dst, retn); | |
963 | break; | |
964 | case 2: | |
965 | __asm_clear_2(dst, retn); | |
966 | break; | |
967 | case 3: | |
968 | __asm_clear_3(dst, retn); | |
969 | break; | |
970 | } | |
971 | ||
972 | return retn; | |
973 | } | |
9da3ee9a | 974 | EXPORT_SYMBOL(__do_clear_user); |
373cd784 JH |
975 | |
976 | unsigned char __get_user_asm_b(const void __user *addr, long *err) | |
977 | { | |
978 | register unsigned char x asm ("D0Re0") = 0; | |
979 | asm volatile ( | |
980 | " GETB %0,[%2]\n" | |
981 | "1:\n" | |
982 | " GETB %0,[%2]\n" | |
983 | "2:\n" | |
984 | " .section .fixup,\"ax\"\n" | |
985 | "3: MOV D0FrT,%3\n" | |
986 | " SETD [%1],D0FrT\n" | |
987 | " MOVT D0FrT,#HI(2b)\n" | |
988 | " JUMP D0FrT,#LO(2b)\n" | |
989 | " .previous\n" | |
990 | " .section __ex_table,\"a\"\n" | |
991 | " .long 1b,3b\n" | |
992 | " .previous\n" | |
993 | : "=r" (x) | |
994 | : "r" (err), "r" (addr), "P" (-EFAULT) | |
995 | : "D0FrT"); | |
996 | return x; | |
997 | } | |
9da3ee9a | 998 | EXPORT_SYMBOL(__get_user_asm_b); |
373cd784 JH |
999 | |
1000 | unsigned short __get_user_asm_w(const void __user *addr, long *err) | |
1001 | { | |
1002 | register unsigned short x asm ("D0Re0") = 0; | |
1003 | asm volatile ( | |
1004 | " GETW %0,[%2]\n" | |
1005 | "1:\n" | |
1006 | " GETW %0,[%2]\n" | |
1007 | "2:\n" | |
1008 | " .section .fixup,\"ax\"\n" | |
1009 | "3: MOV D0FrT,%3\n" | |
1010 | " SETD [%1],D0FrT\n" | |
1011 | " MOVT D0FrT,#HI(2b)\n" | |
1012 | " JUMP D0FrT,#LO(2b)\n" | |
1013 | " .previous\n" | |
1014 | " .section __ex_table,\"a\"\n" | |
1015 | " .long 1b,3b\n" | |
1016 | " .previous\n" | |
1017 | : "=r" (x) | |
1018 | : "r" (err), "r" (addr), "P" (-EFAULT) | |
1019 | : "D0FrT"); | |
1020 | return x; | |
1021 | } | |
9da3ee9a | 1022 | EXPORT_SYMBOL(__get_user_asm_w); |
373cd784 JH |
1023 | |
1024 | unsigned int __get_user_asm_d(const void __user *addr, long *err) | |
1025 | { | |
1026 | register unsigned int x asm ("D0Re0") = 0; | |
1027 | asm volatile ( | |
1028 | " GETD %0,[%2]\n" | |
1029 | "1:\n" | |
1030 | " GETD %0,[%2]\n" | |
1031 | "2:\n" | |
1032 | " .section .fixup,\"ax\"\n" | |
1033 | "3: MOV D0FrT,%3\n" | |
1034 | " SETD [%1],D0FrT\n" | |
1035 | " MOVT D0FrT,#HI(2b)\n" | |
1036 | " JUMP D0FrT,#LO(2b)\n" | |
1037 | " .previous\n" | |
1038 | " .section __ex_table,\"a\"\n" | |
1039 | " .long 1b,3b\n" | |
1040 | " .previous\n" | |
1041 | : "=r" (x) | |
1042 | : "r" (err), "r" (addr), "P" (-EFAULT) | |
1043 | : "D0FrT"); | |
1044 | return x; | |
1045 | } | |
9da3ee9a | 1046 | EXPORT_SYMBOL(__get_user_asm_d); |
373cd784 | 1047 | |
d3ba2e92 JH |
1048 | unsigned long long __get_user_asm_l(const void __user *addr, long *err) |
1049 | { | |
1050 | register unsigned long long x asm ("D0Re0") = 0; | |
1051 | asm volatile ( | |
1052 | " GETL %0,%t0,[%2]\n" | |
1053 | "1:\n" | |
1054 | " GETL %0,%t0,[%2]\n" | |
1055 | "2:\n" | |
1056 | " .section .fixup,\"ax\"\n" | |
1057 | "3: MOV D0FrT,%3\n" | |
1058 | " SETD [%1],D0FrT\n" | |
1059 | " MOVT D0FrT,#HI(2b)\n" | |
1060 | " JUMP D0FrT,#LO(2b)\n" | |
1061 | " .previous\n" | |
1062 | " .section __ex_table,\"a\"\n" | |
1063 | " .long 1b,3b\n" | |
1064 | " .previous\n" | |
1065 | : "=r" (x) | |
1066 | : "r" (err), "r" (addr), "P" (-EFAULT) | |
1067 | : "D0FrT"); | |
1068 | return x; | |
1069 | } | |
1070 | EXPORT_SYMBOL(__get_user_asm_l); | |
1071 | ||
373cd784 JH |
1072 | long __put_user_asm_b(unsigned int x, void __user *addr) |
1073 | { | |
1074 | register unsigned int err asm ("D0Re0") = 0; | |
1075 | asm volatile ( | |
1076 | " MOV %0,#0\n" | |
1077 | " SETB [%2],%1\n" | |
1078 | "1:\n" | |
1079 | " SETB [%2],%1\n" | |
1080 | "2:\n" | |
1081 | ".section .fixup,\"ax\"\n" | |
1082 | "3: MOV %0,%3\n" | |
1083 | " MOVT D0FrT,#HI(2b)\n" | |
1084 | " JUMP D0FrT,#LO(2b)\n" | |
1085 | ".previous\n" | |
1086 | ".section __ex_table,\"a\"\n" | |
1087 | " .long 1b,3b\n" | |
1088 | ".previous" | |
1089 | : "=r"(err) | |
1090 | : "d" (x), "a" (addr), "P"(-EFAULT) | |
1091 | : "D0FrT"); | |
1092 | return err; | |
1093 | } | |
9da3ee9a | 1094 | EXPORT_SYMBOL(__put_user_asm_b); |
373cd784 JH |
1095 | |
1096 | long __put_user_asm_w(unsigned int x, void __user *addr) | |
1097 | { | |
1098 | register unsigned int err asm ("D0Re0") = 0; | |
1099 | asm volatile ( | |
1100 | " MOV %0,#0\n" | |
1101 | " SETW [%2],%1\n" | |
1102 | "1:\n" | |
1103 | " SETW [%2],%1\n" | |
1104 | "2:\n" | |
1105 | ".section .fixup,\"ax\"\n" | |
1106 | "3: MOV %0,%3\n" | |
1107 | " MOVT D0FrT,#HI(2b)\n" | |
1108 | " JUMP D0FrT,#LO(2b)\n" | |
1109 | ".previous\n" | |
1110 | ".section __ex_table,\"a\"\n" | |
1111 | " .long 1b,3b\n" | |
1112 | ".previous" | |
1113 | : "=r"(err) | |
1114 | : "d" (x), "a" (addr), "P"(-EFAULT) | |
1115 | : "D0FrT"); | |
1116 | return err; | |
1117 | } | |
9da3ee9a | 1118 | EXPORT_SYMBOL(__put_user_asm_w); |
373cd784 JH |
1119 | |
1120 | long __put_user_asm_d(unsigned int x, void __user *addr) | |
1121 | { | |
1122 | register unsigned int err asm ("D0Re0") = 0; | |
1123 | asm volatile ( | |
1124 | " MOV %0,#0\n" | |
1125 | " SETD [%2],%1\n" | |
1126 | "1:\n" | |
1127 | " SETD [%2],%1\n" | |
1128 | "2:\n" | |
1129 | ".section .fixup,\"ax\"\n" | |
1130 | "3: MOV %0,%3\n" | |
1131 | " MOVT D0FrT,#HI(2b)\n" | |
1132 | " JUMP D0FrT,#LO(2b)\n" | |
1133 | ".previous\n" | |
1134 | ".section __ex_table,\"a\"\n" | |
1135 | " .long 1b,3b\n" | |
1136 | ".previous" | |
1137 | : "=r"(err) | |
1138 | : "d" (x), "a" (addr), "P"(-EFAULT) | |
1139 | : "D0FrT"); | |
1140 | return err; | |
1141 | } | |
9da3ee9a | 1142 | EXPORT_SYMBOL(__put_user_asm_d); |
373cd784 JH |
1143 | |
1144 | long __put_user_asm_l(unsigned long long x, void __user *addr) | |
1145 | { | |
1146 | register unsigned int err asm ("D0Re0") = 0; | |
1147 | asm volatile ( | |
1148 | " MOV %0,#0\n" | |
1149 | " SETL [%2],%1,%t1\n" | |
1150 | "1:\n" | |
1151 | " SETL [%2],%1,%t1\n" | |
1152 | "2:\n" | |
1153 | ".section .fixup,\"ax\"\n" | |
1154 | "3: MOV %0,%3\n" | |
1155 | " MOVT D0FrT,#HI(2b)\n" | |
1156 | " JUMP D0FrT,#LO(2b)\n" | |
1157 | ".previous\n" | |
1158 | ".section __ex_table,\"a\"\n" | |
1159 | " .long 1b,3b\n" | |
1160 | ".previous" | |
1161 | : "=r"(err) | |
1162 | : "d" (x), "a" (addr), "P"(-EFAULT) | |
1163 | : "D0FrT"); | |
1164 | return err; | |
1165 | } | |
9da3ee9a | 1166 | EXPORT_SYMBOL(__put_user_asm_l); |
373cd784 JH |
1167 | |
1168 | long strnlen_user(const char __user *src, long count) | |
1169 | { | |
1170 | long res; | |
1171 | ||
1172 | if (!access_ok(VERIFY_READ, src, 0)) | |
1173 | return 0; | |
1174 | ||
1175 | asm volatile (" MOV D0Ar4, %1\n" | |
1176 | " MOV D0Ar6, %2\n" | |
1177 | "0:\n" | |
1178 | " SUBS D0FrT, D0Ar6, #0\n" | |
1179 | " SUB D0Ar6, D0Ar6, #1\n" | |
1180 | " BLE 2f\n" | |
1181 | " GETB D0FrT, [D0Ar4+#1++]\n" | |
1182 | "1:\n" | |
1183 | " TST D0FrT, #255\n" | |
1184 | " BNE 0b\n" | |
1185 | "2:\n" | |
1186 | " SUB %0, %2, D0Ar6\n" | |
1187 | "3:\n" | |
1188 | " .section .fixup,\"ax\"\n" | |
1189 | "4:\n" | |
1190 | " MOV %0, #0\n" | |
1191 | " MOVT D0FrT,#HI(3b)\n" | |
1192 | " JUMP D0FrT,#LO(3b)\n" | |
1193 | " .previous\n" | |
1194 | " .section __ex_table,\"a\"\n" | |
1195 | " .long 1b,4b\n" | |
1196 | " .previous\n" | |
1197 | : "=r" (res) | |
1198 | : "r" (src), "r" (count) | |
1199 | : "D0FrT", "D0Ar4", "D0Ar6", "cc"); | |
1200 | ||
1201 | return res; | |
1202 | } | |
9da3ee9a | 1203 | EXPORT_SYMBOL(strnlen_user); |
373cd784 JH |
1204 | |
1205 | long __strncpy_from_user(char *dst, const char __user *src, long count) | |
1206 | { | |
1207 | long res; | |
1208 | ||
1209 | if (count == 0) | |
1210 | return 0; | |
1211 | ||
1212 | /* | |
1213 | * Currently, in 2.4.0-test9, most ports use a simple byte-copy loop. | |
1214 | * So do we. | |
1215 | * | |
1216 | * This code is deduced from: | |
1217 | * | |
1218 | * char tmp2; | |
1219 | * long tmp1, tmp3; | |
1220 | * tmp1 = count; | |
1221 | * while ((*dst++ = (tmp2 = *src++)) != 0 | |
1222 | * && --tmp1) | |
1223 | * ; | |
1224 | * | |
1225 | * res = count - tmp1; | |
1226 | * | |
1227 | * with tweaks. | |
1228 | */ | |
1229 | ||
1230 | asm volatile (" MOV %0,%3\n" | |
1231 | "1:\n" | |
1232 | " GETB D0FrT,[%2++]\n" | |
1233 | "2:\n" | |
1234 | " CMP D0FrT,#0\n" | |
1235 | " SETB [%1++],D0FrT\n" | |
1236 | " BEQ 3f\n" | |
1237 | " SUBS %0,%0,#1\n" | |
1238 | " BNZ 1b\n" | |
1239 | "3:\n" | |
1240 | " SUB %0,%3,%0\n" | |
1241 | "4:\n" | |
1242 | " .section .fixup,\"ax\"\n" | |
1243 | "5:\n" | |
1244 | " MOV %0,%7\n" | |
1245 | " MOVT D0FrT,#HI(4b)\n" | |
1246 | " JUMP D0FrT,#LO(4b)\n" | |
1247 | " .previous\n" | |
1248 | " .section __ex_table,\"a\"\n" | |
1249 | " .long 2b,5b\n" | |
1250 | " .previous" | |
1251 | : "=r" (res), "=r" (dst), "=r" (src), "=r" (count) | |
1252 | : "3" (count), "1" (dst), "2" (src), "P" (-EFAULT) | |
1253 | : "D0FrT", "memory", "cc"); | |
1254 | ||
1255 | return res; | |
1256 | } | |
9da3ee9a | 1257 | EXPORT_SYMBOL(__strncpy_from_user); |