Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * csum_partial_copy - do IP checksumming and copy | |
3 | * | |
4 | * (C) Copyright 1996 Linus Torvalds | |
c3a2ddee | 5 | * accelerated versions (and 21264 assembly versions ) contributed by |
1da177e4 LT |
6 | * Rick Gorton <rick.gorton@alpha-processor.com> |
7 | * | |
8 | * Don't look at this too closely - you'll go mad. The things | |
9 | * we do for performance.. | |
10 | */ | |
11 | ||
12 | #include <linux/types.h> | |
13 | #include <linux/string.h> | |
7c0f6ba6 | 14 | #include <linux/uaccess.h> |
1da177e4 LT |
15 | |
16 | ||
17 | #define ldq_u(x,y) \ | |
18 | __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y))) | |
19 | ||
20 | #define stq_u(x,y) \ | |
21 | __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x)) | |
22 | ||
23 | #define extql(x,y,z) \ | |
24 | __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y)) | |
25 | ||
26 | #define extqh(x,y,z) \ | |
27 | __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) | |
28 | ||
29 | #define mskql(x,y,z) \ | |
30 | __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y)) | |
31 | ||
32 | #define mskqh(x,y,z) \ | |
33 | __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) | |
34 | ||
35 | #define insql(x,y,z) \ | |
36 | __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y)) | |
37 | ||
38 | #define insqh(x,y,z) \ | |
39 | __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) | |
40 | ||
41 | ||
42 | #define __get_user_u(x,ptr) \ | |
43 | ({ \ | |
44 | long __guu_err; \ | |
45 | __asm__ __volatile__( \ | |
46 | "1: ldq_u %0,%2\n" \ | |
47 | "2:\n" \ | |
ca282f69 | 48 | EXC(1b,2b,%0,%1) \ |
1da177e4 LT |
49 | : "=r"(x), "=r"(__guu_err) \ |
50 | : "m"(__m(ptr)), "1"(0)); \ | |
51 | __guu_err; \ | |
52 | }) | |
53 | ||
54 | #define __put_user_u(x,ptr) \ | |
55 | ({ \ | |
56 | long __puu_err; \ | |
57 | __asm__ __volatile__( \ | |
58 | "1: stq_u %2,%1\n" \ | |
59 | "2:\n" \ | |
ca282f69 | 60 | EXC(1b,2b,$31,%0) \ |
1da177e4 LT |
61 | : "=r"(__puu_err) \ |
62 | : "m"(__m(addr)), "rJ"(x), "0"(0)); \ | |
63 | __puu_err; \ | |
64 | }) | |
65 | ||
66 | ||
67 | static inline unsigned short from64to16(unsigned long x) | |
68 | { | |
69 | /* Using extract instructions is a bit more efficient | |
70 | than the original shift/bitmask version. */ | |
71 | ||
72 | union { | |
73 | unsigned long ul; | |
74 | unsigned int ui[2]; | |
75 | unsigned short us[4]; | |
76 | } in_v, tmp_v, out_v; | |
77 | ||
78 | in_v.ul = x; | |
79 | tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1]; | |
80 | ||
81 | /* Since the bits of tmp_v.sh[3] are going to always be zero, | |
82 | we don't have to bother to add that in. */ | |
83 | out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1] | |
84 | + (unsigned long) tmp_v.us[2]; | |
85 | ||
86 | /* Similarly, out_v.us[2] is always zero for the final add. */ | |
87 | return out_v.us[0] + out_v.us[1]; | |
88 | } | |
89 | ||
90 | ||
91 | ||
92 | /* | |
93 | * Ok. This isn't fun, but this is the EASY case. | |
94 | */ | |
95 | static inline unsigned long | |
96 | csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst, | |
97 | long len, unsigned long checksum, | |
98 | int *errp) | |
99 | { | |
100 | unsigned long carry = 0; | |
101 | int err = 0; | |
102 | ||
103 | while (len >= 0) { | |
104 | unsigned long word; | |
105 | err |= __get_user(word, src); | |
106 | checksum += carry; | |
107 | src++; | |
108 | checksum += word; | |
109 | len -= 8; | |
110 | carry = checksum < word; | |
111 | *dst = word; | |
112 | dst++; | |
113 | } | |
114 | len += 8; | |
115 | checksum += carry; | |
116 | if (len) { | |
117 | unsigned long word, tmp; | |
118 | err |= __get_user(word, src); | |
119 | tmp = *dst; | |
120 | mskql(word, len, word); | |
121 | checksum += word; | |
122 | mskqh(tmp, len, tmp); | |
123 | carry = checksum < word; | |
124 | *dst = word | tmp; | |
125 | checksum += carry; | |
126 | } | |
5cfe8f1b | 127 | if (err && errp) *errp = err; |
1da177e4 LT |
128 | return checksum; |
129 | } | |
130 | ||
131 | /* | |
132 | * This is even less fun, but this is still reasonably | |
133 | * easy. | |
134 | */ | |
135 | static inline unsigned long | |
136 | csum_partial_cfu_dest_aligned(const unsigned long __user *src, | |
137 | unsigned long *dst, | |
138 | unsigned long soff, | |
139 | long len, unsigned long checksum, | |
140 | int *errp) | |
141 | { | |
142 | unsigned long first; | |
143 | unsigned long word, carry; | |
144 | unsigned long lastsrc = 7+len+(unsigned long)src; | |
145 | int err = 0; | |
146 | ||
147 | err |= __get_user_u(first,src); | |
148 | carry = 0; | |
149 | while (len >= 0) { | |
150 | unsigned long second; | |
151 | ||
152 | err |= __get_user_u(second, src+1); | |
153 | extql(first, soff, word); | |
154 | len -= 8; | |
155 | src++; | |
156 | extqh(second, soff, first); | |
157 | checksum += carry; | |
158 | word |= first; | |
159 | first = second; | |
160 | checksum += word; | |
161 | *dst = word; | |
162 | dst++; | |
163 | carry = checksum < word; | |
164 | } | |
165 | len += 8; | |
166 | checksum += carry; | |
167 | if (len) { | |
168 | unsigned long tmp; | |
169 | unsigned long second; | |
170 | err |= __get_user_u(second, lastsrc); | |
171 | tmp = *dst; | |
172 | extql(first, soff, word); | |
173 | extqh(second, soff, first); | |
174 | word |= first; | |
175 | mskql(word, len, word); | |
176 | checksum += word; | |
177 | mskqh(tmp, len, tmp); | |
178 | carry = checksum < word; | |
179 | *dst = word | tmp; | |
180 | checksum += carry; | |
181 | } | |
5cfe8f1b | 182 | if (err && errp) *errp = err; |
1da177e4 LT |
183 | return checksum; |
184 | } | |
185 | ||
186 | /* | |
187 | * This is slightly less fun than the above.. | |
188 | */ | |
189 | static inline unsigned long | |
190 | csum_partial_cfu_src_aligned(const unsigned long __user *src, | |
191 | unsigned long *dst, | |
192 | unsigned long doff, | |
193 | long len, unsigned long checksum, | |
194 | unsigned long partial_dest, | |
195 | int *errp) | |
196 | { | |
197 | unsigned long carry = 0; | |
198 | unsigned long word; | |
199 | unsigned long second_dest; | |
200 | int err = 0; | |
201 | ||
202 | mskql(partial_dest, doff, partial_dest); | |
203 | while (len >= 0) { | |
204 | err |= __get_user(word, src); | |
205 | len -= 8; | |
206 | insql(word, doff, second_dest); | |
207 | checksum += carry; | |
208 | stq_u(partial_dest | second_dest, dst); | |
209 | src++; | |
210 | checksum += word; | |
211 | insqh(word, doff, partial_dest); | |
212 | carry = checksum < word; | |
213 | dst++; | |
214 | } | |
215 | len += 8; | |
216 | if (len) { | |
217 | checksum += carry; | |
218 | err |= __get_user(word, src); | |
219 | mskql(word, len, word); | |
220 | len -= 8; | |
221 | checksum += word; | |
222 | insql(word, doff, second_dest); | |
223 | len += doff; | |
224 | carry = checksum < word; | |
225 | partial_dest |= second_dest; | |
226 | if (len >= 0) { | |
227 | stq_u(partial_dest, dst); | |
228 | if (!len) goto out; | |
229 | dst++; | |
230 | insqh(word, doff, partial_dest); | |
231 | } | |
232 | doff = len; | |
233 | } | |
234 | ldq_u(second_dest, dst); | |
235 | mskqh(second_dest, doff, second_dest); | |
236 | stq_u(partial_dest | second_dest, dst); | |
237 | out: | |
238 | checksum += carry; | |
5cfe8f1b | 239 | if (err && errp) *errp = err; |
1da177e4 LT |
240 | return checksum; |
241 | } | |
242 | ||
243 | /* | |
244 | * This is so totally un-fun that it's frightening. Don't | |
245 | * look at this too closely, you'll go blind. | |
246 | */ | |
247 | static inline unsigned long | |
248 | csum_partial_cfu_unaligned(const unsigned long __user * src, | |
249 | unsigned long * dst, | |
250 | unsigned long soff, unsigned long doff, | |
251 | long len, unsigned long checksum, | |
252 | unsigned long partial_dest, | |
253 | int *errp) | |
254 | { | |
255 | unsigned long carry = 0; | |
256 | unsigned long first; | |
257 | unsigned long lastsrc; | |
258 | int err = 0; | |
259 | ||
260 | err |= __get_user_u(first, src); | |
261 | lastsrc = 7+len+(unsigned long)src; | |
262 | mskql(partial_dest, doff, partial_dest); | |
263 | while (len >= 0) { | |
264 | unsigned long second, word; | |
265 | unsigned long second_dest; | |
266 | ||
267 | err |= __get_user_u(second, src+1); | |
268 | extql(first, soff, word); | |
269 | checksum += carry; | |
270 | len -= 8; | |
271 | extqh(second, soff, first); | |
272 | src++; | |
273 | word |= first; | |
274 | first = second; | |
275 | insql(word, doff, second_dest); | |
276 | checksum += word; | |
277 | stq_u(partial_dest | second_dest, dst); | |
278 | carry = checksum < word; | |
279 | insqh(word, doff, partial_dest); | |
280 | dst++; | |
281 | } | |
282 | len += doff; | |
283 | checksum += carry; | |
284 | if (len >= 0) { | |
285 | unsigned long second, word; | |
286 | unsigned long second_dest; | |
287 | ||
288 | err |= __get_user_u(second, lastsrc); | |
289 | extql(first, soff, word); | |
290 | extqh(second, soff, first); | |
291 | word |= first; | |
292 | first = second; | |
293 | mskql(word, len-doff, word); | |
294 | checksum += word; | |
295 | insql(word, doff, second_dest); | |
296 | carry = checksum < word; | |
297 | stq_u(partial_dest | second_dest, dst); | |
298 | if (len) { | |
299 | ldq_u(second_dest, dst+1); | |
300 | insqh(word, doff, partial_dest); | |
301 | mskqh(second_dest, len, second_dest); | |
302 | stq_u(partial_dest | second_dest, dst+1); | |
303 | } | |
304 | checksum += carry; | |
305 | } else { | |
306 | unsigned long second, word; | |
307 | unsigned long second_dest; | |
308 | ||
309 | err |= __get_user_u(second, lastsrc); | |
310 | extql(first, soff, word); | |
311 | extqh(second, soff, first); | |
312 | word |= first; | |
313 | ldq_u(second_dest, dst); | |
314 | mskql(word, len-doff, word); | |
315 | checksum += word; | |
316 | mskqh(second_dest, len, second_dest); | |
317 | carry = checksum < word; | |
318 | insql(word, doff, word); | |
319 | stq_u(partial_dest | word | second_dest, dst); | |
320 | checksum += carry; | |
321 | } | |
5cfe8f1b | 322 | if (err && errp) *errp = err; |
1da177e4 LT |
323 | return checksum; |
324 | } | |
325 | ||
9be259aa AV |
326 | __wsum |
327 | csum_partial_copy_from_user(const void __user *src, void *dst, int len, | |
328 | __wsum sum, int *errp) | |
1da177e4 | 329 | { |
9be259aa | 330 | unsigned long checksum = (__force u32) sum; |
1da177e4 LT |
331 | unsigned long soff = 7 & (unsigned long) src; |
332 | unsigned long doff = 7 & (unsigned long) dst; | |
333 | ||
334 | if (len) { | |
3ddc5b46 | 335 | if (!access_ok(VERIFY_READ, src, len)) { |
5cfe8f1b | 336 | if (errp) *errp = -EFAULT; |
3ddc5b46 MD |
337 | memset(dst, 0, len); |
338 | return sum; | |
339 | } | |
1da177e4 LT |
340 | if (!doff) { |
341 | if (!soff) | |
342 | checksum = csum_partial_cfu_aligned( | |
343 | (const unsigned long __user *) src, | |
344 | (unsigned long *) dst, | |
345 | len-8, checksum, errp); | |
346 | else | |
347 | checksum = csum_partial_cfu_dest_aligned( | |
348 | (const unsigned long __user *) src, | |
349 | (unsigned long *) dst, | |
350 | soff, len-8, checksum, errp); | |
351 | } else { | |
352 | unsigned long partial_dest; | |
353 | ldq_u(partial_dest, dst); | |
354 | if (!soff) | |
355 | checksum = csum_partial_cfu_src_aligned( | |
356 | (const unsigned long __user *) src, | |
357 | (unsigned long *) dst, | |
358 | doff, len-8, checksum, | |
359 | partial_dest, errp); | |
360 | else | |
361 | checksum = csum_partial_cfu_unaligned( | |
362 | (const unsigned long __user *) src, | |
363 | (unsigned long *) dst, | |
364 | soff, doff, len-8, checksum, | |
365 | partial_dest, errp); | |
366 | } | |
367 | checksum = from64to16 (checksum); | |
368 | } | |
9be259aa | 369 | return (__force __wsum)checksum; |
1da177e4 | 370 | } |
00fc0e0d | 371 | EXPORT_SYMBOL(csum_partial_copy_from_user); |
1da177e4 | 372 | |
9be259aa AV |
373 | __wsum |
374 | csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum) | |
1da177e4 | 375 | { |
0ef38d70 MP |
376 | __wsum checksum; |
377 | mm_segment_t oldfs = get_fs(); | |
378 | set_fs(KERNEL_DS); | |
379 | checksum = csum_partial_copy_from_user((__force const void __user *)src, | |
380 | dst, len, sum, NULL); | |
381 | set_fs(oldfs); | |
382 | return checksum; | |
1da177e4 | 383 | } |
00fc0e0d | 384 | EXPORT_SYMBOL(csum_partial_copy_nocheck); |