Commit | Line | Data |
---|---|---|
59f35d53 | 1 | /* |
c1821c2e GS |
2 | * User access functions based on page table walks for enhanced |
3 | * system layout without hardware support. | |
59f35d53 GS |
4 | * |
5 | * Copyright IBM Corp. 2006 | |
6 | * Author(s): Gerald Schaefer (gerald.schaefer@de.ibm.com) | |
7 | */ | |
8 | ||
9 | #include <linux/errno.h> | |
d8ad075e | 10 | #include <linux/hardirq.h> |
59f35d53 | 11 | #include <linux/mm.h> |
22155914 | 12 | #include <asm/uaccess.h> |
59f35d53 | 13 | #include <asm/futex.h> |
2b67fc46 | 14 | #include "uaccess.h" |
59f35d53 | 15 | |
e4aa402e MS |
16 | static inline pte_t *follow_table(struct mm_struct *mm, unsigned long addr) |
17 | { | |
18 | pgd_t *pgd; | |
190a1d72 | 19 | pud_t *pud; |
e4aa402e MS |
20 | pmd_t *pmd; |
21 | ||
22 | pgd = pgd_offset(mm, addr); | |
23 | if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) | |
6c1e3e79 | 24 | return (pte_t *) 0x3a; |
e4aa402e | 25 | |
190a1d72 MS |
26 | pud = pud_offset(pgd, addr); |
27 | if (pud_none(*pud) || unlikely(pud_bad(*pud))) | |
6c1e3e79 | 28 | return (pte_t *) 0x3b; |
190a1d72 MS |
29 | |
30 | pmd = pmd_offset(pud, addr); | |
e4aa402e | 31 | if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) |
6c1e3e79 | 32 | return (pte_t *) 0x10; |
e4aa402e MS |
33 | |
34 | return pte_offset_map(pmd, addr); | |
35 | } | |
36 | ||
6c1e3e79 GS |
37 | static __always_inline size_t __user_copy_pt(unsigned long uaddr, void *kptr, |
38 | size_t n, int write_user) | |
59f35d53 GS |
39 | { |
40 | struct mm_struct *mm = current->mm; | |
41 | unsigned long offset, pfn, done, size; | |
59f35d53 GS |
42 | pte_t *pte; |
43 | void *from, *to; | |
44 | ||
45 | done = 0; | |
46 | retry: | |
47 | spin_lock(&mm->page_table_lock); | |
48 | do { | |
e4aa402e | 49 | pte = follow_table(mm, uaddr); |
6c1e3e79 GS |
50 | if ((unsigned long) pte < 0x1000) |
51 | goto fault; | |
52 | if (!pte_present(*pte)) { | |
53 | pte = (pte_t *) 0x11; | |
59f35d53 | 54 | goto fault; |
6c1e3e79 GS |
55 | } else if (write_user && !pte_write(*pte)) { |
56 | pte = (pte_t *) 0x04; | |
57 | goto fault; | |
58 | } | |
59f35d53 GS |
59 | |
60 | pfn = pte_pfn(*pte); | |
59f35d53 GS |
61 | offset = uaddr & (PAGE_SIZE - 1); |
62 | size = min(n - done, PAGE_SIZE - offset); | |
63 | if (write_user) { | |
64 | to = (void *)((pfn << PAGE_SHIFT) + offset); | |
65 | from = kptr + done; | |
66 | } else { | |
67 | from = (void *)((pfn << PAGE_SHIFT) + offset); | |
68 | to = kptr + done; | |
69 | } | |
70 | memcpy(to, from, size); | |
71 | done += size; | |
72 | uaddr += size; | |
73 | } while (done < n); | |
59f35d53 GS |
74 | spin_unlock(&mm->page_table_lock); |
75 | return n - done; | |
76 | fault: | |
77 | spin_unlock(&mm->page_table_lock); | |
6c1e3e79 | 78 | if (__handle_fault(uaddr, (unsigned long) pte, write_user)) |
59f35d53 GS |
79 | return n - done; |
80 | goto retry; | |
81 | } | |
82 | ||
c1821c2e GS |
83 | /* |
84 | * Do DAT for user address by page table walk, return kernel address. | |
85 | * This function needs to be called with current->mm->page_table_lock held. | |
86 | */ | |
6c1e3e79 | 87 | static __always_inline unsigned long __dat_user_addr(unsigned long uaddr) |
c1821c2e GS |
88 | { |
89 | struct mm_struct *mm = current->mm; | |
6c1e3e79 | 90 | unsigned long pfn; |
c1821c2e GS |
91 | pte_t *pte; |
92 | int rc; | |
93 | ||
c1821c2e | 94 | retry: |
e4aa402e | 95 | pte = follow_table(mm, uaddr); |
6c1e3e79 GS |
96 | if ((unsigned long) pte < 0x1000) |
97 | goto fault; | |
98 | if (!pte_present(*pte)) { | |
99 | pte = (pte_t *) 0x11; | |
c1821c2e | 100 | goto fault; |
6c1e3e79 | 101 | } |
c1821c2e GS |
102 | |
103 | pfn = pte_pfn(*pte); | |
6c1e3e79 | 104 | return (pfn << PAGE_SHIFT) + (uaddr & (PAGE_SIZE - 1)); |
c1821c2e GS |
105 | fault: |
106 | spin_unlock(&mm->page_table_lock); | |
6c1e3e79 | 107 | rc = __handle_fault(uaddr, (unsigned long) pte, 0); |
c1821c2e | 108 | spin_lock(&mm->page_table_lock); |
6c1e3e79 GS |
109 | if (!rc) |
110 | goto retry; | |
111 | return 0; | |
c1821c2e GS |
112 | } |
113 | ||
59f35d53 GS |
114 | size_t copy_from_user_pt(size_t n, const void __user *from, void *to) |
115 | { | |
116 | size_t rc; | |
117 | ||
118 | if (segment_eq(get_fs(), KERNEL_DS)) { | |
119 | memcpy(to, (void __kernel __force *) from, n); | |
120 | return 0; | |
121 | } | |
122 | rc = __user_copy_pt((unsigned long) from, to, n, 0); | |
123 | if (unlikely(rc)) | |
124 | memset(to + n - rc, 0, rc); | |
125 | return rc; | |
126 | } | |
127 | ||
128 | size_t copy_to_user_pt(size_t n, void __user *to, const void *from) | |
129 | { | |
130 | if (segment_eq(get_fs(), KERNEL_DS)) { | |
131 | memcpy((void __kernel __force *) to, from, n); | |
132 | return 0; | |
133 | } | |
134 | return __user_copy_pt((unsigned long) to, (void *) from, n, 1); | |
135 | } | |
c1821c2e GS |
136 | |
137 | static size_t clear_user_pt(size_t n, void __user *to) | |
138 | { | |
139 | long done, size, ret; | |
140 | ||
141 | if (segment_eq(get_fs(), KERNEL_DS)) { | |
142 | memset((void __kernel __force *) to, 0, n); | |
143 | return 0; | |
144 | } | |
145 | done = 0; | |
146 | do { | |
147 | if (n - done > PAGE_SIZE) | |
148 | size = PAGE_SIZE; | |
149 | else | |
150 | size = n - done; | |
151 | ret = __user_copy_pt((unsigned long) to + done, | |
152 | &empty_zero_page, size, 1); | |
153 | done += size; | |
154 | if (ret) | |
155 | return ret + n - done; | |
156 | } while (done < n); | |
157 | return 0; | |
158 | } | |
159 | ||
160 | static size_t strnlen_user_pt(size_t count, const char __user *src) | |
161 | { | |
162 | char *addr; | |
163 | unsigned long uaddr = (unsigned long) src; | |
164 | struct mm_struct *mm = current->mm; | |
165 | unsigned long offset, pfn, done, len; | |
c1821c2e GS |
166 | pte_t *pte; |
167 | size_t len_str; | |
168 | ||
169 | if (segment_eq(get_fs(), KERNEL_DS)) | |
170 | return strnlen((const char __kernel __force *) src, count) + 1; | |
171 | done = 0; | |
172 | retry: | |
173 | spin_lock(&mm->page_table_lock); | |
174 | do { | |
e4aa402e | 175 | pte = follow_table(mm, uaddr); |
6c1e3e79 | 176 | if ((unsigned long) pte < 0x1000) |
c1821c2e | 177 | goto fault; |
6c1e3e79 GS |
178 | if (!pte_present(*pte)) { |
179 | pte = (pte_t *) 0x11; | |
180 | goto fault; | |
181 | } | |
c1821c2e GS |
182 | |
183 | pfn = pte_pfn(*pte); | |
c1821c2e GS |
184 | offset = uaddr & (PAGE_SIZE-1); |
185 | addr = (char *)(pfn << PAGE_SHIFT) + offset; | |
186 | len = min(count - done, PAGE_SIZE - offset); | |
187 | len_str = strnlen(addr, len); | |
188 | done += len_str; | |
189 | uaddr += len_str; | |
190 | } while ((len_str == len) && (done < count)); | |
c1821c2e GS |
191 | spin_unlock(&mm->page_table_lock); |
192 | return done + 1; | |
193 | fault: | |
194 | spin_unlock(&mm->page_table_lock); | |
6c1e3e79 | 195 | if (__handle_fault(uaddr, (unsigned long) pte, 0)) |
c1821c2e | 196 | return 0; |
c1821c2e GS |
197 | goto retry; |
198 | } | |
199 | ||
200 | static size_t strncpy_from_user_pt(size_t count, const char __user *src, | |
201 | char *dst) | |
202 | { | |
203 | size_t n = strnlen_user_pt(count, src); | |
204 | ||
205 | if (!n) | |
206 | return -EFAULT; | |
207 | if (n > count) | |
208 | n = count; | |
209 | if (segment_eq(get_fs(), KERNEL_DS)) { | |
210 | memcpy(dst, (const char __kernel __force *) src, n); | |
211 | if (dst[n-1] == '\0') | |
212 | return n-1; | |
213 | else | |
214 | return n; | |
215 | } | |
216 | if (__user_copy_pt((unsigned long) src, dst, n, 0)) | |
217 | return -EFAULT; | |
218 | if (dst[n-1] == '\0') | |
219 | return n-1; | |
220 | else | |
221 | return n; | |
222 | } | |
223 | ||
224 | static size_t copy_in_user_pt(size_t n, void __user *to, | |
225 | const void __user *from) | |
226 | { | |
227 | struct mm_struct *mm = current->mm; | |
228 | unsigned long offset_from, offset_to, offset_max, pfn_from, pfn_to, | |
6c1e3e79 | 229 | uaddr, done, size, error_code; |
c1821c2e GS |
230 | unsigned long uaddr_from = (unsigned long) from; |
231 | unsigned long uaddr_to = (unsigned long) to; | |
c1821c2e GS |
232 | pte_t *pte_from, *pte_to; |
233 | int write_user; | |
234 | ||
3f12ebce HC |
235 | if (segment_eq(get_fs(), KERNEL_DS)) { |
236 | memcpy((void __force *) to, (void __force *) from, n); | |
237 | return 0; | |
238 | } | |
c1821c2e GS |
239 | done = 0; |
240 | retry: | |
241 | spin_lock(&mm->page_table_lock); | |
242 | do { | |
6c1e3e79 GS |
243 | write_user = 0; |
244 | uaddr = uaddr_from; | |
e4aa402e | 245 | pte_from = follow_table(mm, uaddr_from); |
6c1e3e79 GS |
246 | error_code = (unsigned long) pte_from; |
247 | if (error_code < 0x1000) | |
248 | goto fault; | |
249 | if (!pte_present(*pte_from)) { | |
250 | error_code = 0x11; | |
c1821c2e GS |
251 | goto fault; |
252 | } | |
e4aa402e | 253 | |
6c1e3e79 GS |
254 | write_user = 1; |
255 | uaddr = uaddr_to; | |
e4aa402e | 256 | pte_to = follow_table(mm, uaddr_to); |
6c1e3e79 GS |
257 | error_code = (unsigned long) pte_to; |
258 | if (error_code < 0x1000) | |
259 | goto fault; | |
260 | if (!pte_present(*pte_to)) { | |
261 | error_code = 0x11; | |
262 | goto fault; | |
263 | } else if (!pte_write(*pte_to)) { | |
264 | error_code = 0x04; | |
c1821c2e GS |
265 | goto fault; |
266 | } | |
267 | ||
268 | pfn_from = pte_pfn(*pte_from); | |
c1821c2e | 269 | pfn_to = pte_pfn(*pte_to); |
c1821c2e GS |
270 | offset_from = uaddr_from & (PAGE_SIZE-1); |
271 | offset_to = uaddr_from & (PAGE_SIZE-1); | |
272 | offset_max = max(offset_from, offset_to); | |
273 | size = min(n - done, PAGE_SIZE - offset_max); | |
274 | ||
275 | memcpy((void *)(pfn_to << PAGE_SHIFT) + offset_to, | |
276 | (void *)(pfn_from << PAGE_SHIFT) + offset_from, size); | |
277 | done += size; | |
278 | uaddr_from += size; | |
279 | uaddr_to += size; | |
280 | } while (done < n); | |
c1821c2e GS |
281 | spin_unlock(&mm->page_table_lock); |
282 | return n - done; | |
283 | fault: | |
284 | spin_unlock(&mm->page_table_lock); | |
6c1e3e79 | 285 | if (__handle_fault(uaddr, error_code, write_user)) |
c1821c2e GS |
286 | return n - done; |
287 | goto retry; | |
288 | } | |
289 | ||
290 | #define __futex_atomic_op(insn, ret, oldval, newval, uaddr, oparg) \ | |
291 | asm volatile("0: l %1,0(%6)\n" \ | |
292 | "1: " insn \ | |
293 | "2: cs %1,%2,0(%6)\n" \ | |
294 | "3: jl 1b\n" \ | |
295 | " lhi %0,0\n" \ | |
296 | "4:\n" \ | |
297 | EX_TABLE(0b,4b) EX_TABLE(2b,4b) EX_TABLE(3b,4b) \ | |
298 | : "=d" (ret), "=&d" (oldval), "=&d" (newval), \ | |
299 | "=m" (*uaddr) \ | |
300 | : "0" (-EFAULT), "d" (oparg), "a" (uaddr), \ | |
301 | "m" (*uaddr) : "cc" ); | |
302 | ||
8d7718aa | 303 | static int __futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old) |
c1821c2e GS |
304 | { |
305 | int oldval = 0, newval, ret; | |
306 | ||
c1821c2e GS |
307 | switch (op) { |
308 | case FUTEX_OP_SET: | |
309 | __futex_atomic_op("lr %2,%5\n", | |
310 | ret, oldval, newval, uaddr, oparg); | |
311 | break; | |
312 | case FUTEX_OP_ADD: | |
313 | __futex_atomic_op("lr %2,%1\nar %2,%5\n", | |
314 | ret, oldval, newval, uaddr, oparg); | |
315 | break; | |
316 | case FUTEX_OP_OR: | |
317 | __futex_atomic_op("lr %2,%1\nor %2,%5\n", | |
318 | ret, oldval, newval, uaddr, oparg); | |
319 | break; | |
320 | case FUTEX_OP_ANDN: | |
321 | __futex_atomic_op("lr %2,%1\nnr %2,%5\n", | |
322 | ret, oldval, newval, uaddr, oparg); | |
323 | break; | |
324 | case FUTEX_OP_XOR: | |
325 | __futex_atomic_op("lr %2,%1\nxr %2,%5\n", | |
326 | ret, oldval, newval, uaddr, oparg); | |
327 | break; | |
328 | default: | |
329 | ret = -ENOSYS; | |
330 | } | |
3f12ebce HC |
331 | if (ret == 0) |
332 | *old = oldval; | |
c1821c2e GS |
333 | return ret; |
334 | } | |
335 | ||
8d7718aa | 336 | int futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old) |
c1821c2e GS |
337 | { |
338 | int ret; | |
339 | ||
3f12ebce HC |
340 | if (segment_eq(get_fs(), KERNEL_DS)) |
341 | return __futex_atomic_op_pt(op, uaddr, oparg, old); | |
c1821c2e | 342 | spin_lock(¤t->mm->page_table_lock); |
3c52e49d MS |
343 | uaddr = (u32 __force __user *) |
344 | __dat_user_addr((__force unsigned long) uaddr); | |
c1821c2e GS |
345 | if (!uaddr) { |
346 | spin_unlock(¤t->mm->page_table_lock); | |
347 | return -EFAULT; | |
348 | } | |
349 | get_page(virt_to_page(uaddr)); | |
350 | spin_unlock(¤t->mm->page_table_lock); | |
3f12ebce HC |
351 | ret = __futex_atomic_op_pt(op, uaddr, oparg, old); |
352 | put_page(virt_to_page(uaddr)); | |
353 | return ret; | |
354 | } | |
355 | ||
8d7718aa ML |
356 | static int __futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr, |
357 | u32 oldval, u32 newval) | |
3f12ebce HC |
358 | { |
359 | int ret; | |
360 | ||
361 | asm volatile("0: cs %1,%4,0(%5)\n" | |
37a9d912 | 362 | "1: la %0,0\n" |
3f12ebce HC |
363 | "2:\n" |
364 | EX_TABLE(0b,2b) EX_TABLE(1b,2b) | |
c1821c2e GS |
365 | : "=d" (ret), "+d" (oldval), "=m" (*uaddr) |
366 | : "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr) | |
367 | : "cc", "memory" ); | |
37a9d912 | 368 | *uval = oldval; |
3f12ebce HC |
369 | return ret; |
370 | } | |
371 | ||
8d7718aa ML |
372 | int futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr, |
373 | u32 oldval, u32 newval) | |
3f12ebce HC |
374 | { |
375 | int ret; | |
376 | ||
377 | if (segment_eq(get_fs(), KERNEL_DS)) | |
37a9d912 | 378 | return __futex_atomic_cmpxchg_pt(uval, uaddr, oldval, newval); |
3f12ebce | 379 | spin_lock(¤t->mm->page_table_lock); |
3c52e49d MS |
380 | uaddr = (u32 __force __user *) |
381 | __dat_user_addr((__force unsigned long) uaddr); | |
3f12ebce HC |
382 | if (!uaddr) { |
383 | spin_unlock(¤t->mm->page_table_lock); | |
384 | return -EFAULT; | |
385 | } | |
386 | get_page(virt_to_page(uaddr)); | |
387 | spin_unlock(¤t->mm->page_table_lock); | |
37a9d912 | 388 | ret = __futex_atomic_cmpxchg_pt(uval, uaddr, oldval, newval); |
c1821c2e GS |
389 | put_page(virt_to_page(uaddr)); |
390 | return ret; | |
391 | } | |
392 | ||
393 | struct uaccess_ops uaccess_pt = { | |
394 | .copy_from_user = copy_from_user_pt, | |
395 | .copy_from_user_small = copy_from_user_pt, | |
396 | .copy_to_user = copy_to_user_pt, | |
397 | .copy_to_user_small = copy_to_user_pt, | |
398 | .copy_in_user = copy_in_user_pt, | |
399 | .clear_user = clear_user_pt, | |
400 | .strnlen_user = strnlen_user_pt, | |
401 | .strncpy_from_user = strncpy_from_user_pt, | |
402 | .futex_atomic_op = futex_atomic_op_pt, | |
403 | .futex_atomic_cmpxchg = futex_atomic_cmpxchg_pt, | |
404 | }; |