Commit | Line | Data |
---|---|---|
0aea86a2 CM |
1 | /* |
2 | * Based on arch/arm/include/asm/uaccess.h | |
3 | * | |
4 | * Copyright (C) 2012 ARM Ltd. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 as | |
8 | * published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License | |
16 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
17 | */ | |
18 | #ifndef __ASM_UACCESS_H | |
19 | #define __ASM_UACCESS_H | |
20 | ||
bd38967d | 21 | #include <asm/alternative.h> |
4b65a5db | 22 | #include <asm/kernel-pgtable.h> |
bd38967d CM |
23 | #include <asm/sysreg.h> |
24 | ||
0aea86a2 CM |
25 | /* |
26 | * User space memory access functions | |
27 | */ | |
87261d19 | 28 | #include <linux/bitops.h> |
bffe1baf | 29 | #include <linux/kasan-checks.h> |
0aea86a2 CM |
30 | #include <linux/string.h> |
31 | #include <linux/thread_info.h> | |
32 | ||
338d4f49 | 33 | #include <asm/cpufeature.h> |
0aea86a2 CM |
34 | #include <asm/ptrace.h> |
35 | #include <asm/errno.h> | |
36 | #include <asm/memory.h> | |
37 | #include <asm/compiler.h> | |
38 | ||
39 | #define VERIFY_READ 0 | |
40 | #define VERIFY_WRITE 1 | |
41 | ||
42 | /* | |
6c94f27a AB |
43 | * The exception table consists of pairs of relative offsets: the first |
44 | * is the relative offset to an instruction that is allowed to fault, | |
45 | * and the second is the relative offset at which the program should | |
46 | * continue. No registers are modified, so it is entirely up to the | |
47 | * continuation code to figure out what to do. | |
0aea86a2 CM |
48 | * |
49 | * All the routines below use bits of fixup code that are out of line | |
50 | * with the main instruction path. This means when everything is well, | |
51 | * we don't even have to jump over them. Further, they do not intrude | |
52 | * on our cache or tlb entries. | |
53 | */ | |
54 | ||
55 | struct exception_table_entry | |
56 | { | |
6c94f27a | 57 | int insn, fixup; |
0aea86a2 CM |
58 | }; |
59 | ||
6c94f27a AB |
60 | #define ARCH_HAS_RELATIVE_EXTABLE |
61 | ||
0aea86a2 CM |
62 | extern int fixup_exception(struct pt_regs *regs); |
63 | ||
64 | #define KERNEL_DS (-1UL) | |
65 | #define get_ds() (KERNEL_DS) | |
66 | ||
67 | #define USER_DS TASK_SIZE_64 | |
68 | #define get_fs() (current_thread_info()->addr_limit) | |
69 | ||
70 | static inline void set_fs(mm_segment_t fs) | |
71 | { | |
72 | current_thread_info()->addr_limit = fs; | |
57f4959b JM |
73 | |
74 | /* | |
75 | * Enable/disable UAO so that copy_to_user() etc can access | |
76 | * kernel memory with the unprivileged instructions. | |
77 | */ | |
78 | if (IS_ENABLED(CONFIG_ARM64_UAO) && fs == KERNEL_DS) | |
79 | asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO)); | |
80 | else | |
81 | asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO, | |
82 | CONFIG_ARM64_UAO)); | |
0aea86a2 CM |
83 | } |
84 | ||
967f0e5d | 85 | #define segment_eq(a, b) ((a) == (b)) |
0aea86a2 | 86 | |
0aea86a2 CM |
87 | /* |
88 | * Test whether a block of memory is a valid user space address. | |
89 | * Returns 1 if the range is valid, 0 otherwise. | |
90 | * | |
91 | * This is equivalent to the following test: | |
31b1e940 | 92 | * (u65)addr + (u65)size <= current->addr_limit |
0aea86a2 CM |
93 | * |
94 | * This needs 65-bit arithmetic. | |
95 | */ | |
96 | #define __range_ok(addr, size) \ | |
97 | ({ \ | |
98 | unsigned long flag, roksum; \ | |
99 | __chk_user_ptr(addr); \ | |
31b1e940 | 100 | asm("adds %1, %1, %3; ccmp %1, %4, #2, cc; cset %0, ls" \ |
0aea86a2 CM |
101 | : "=&r" (flag), "=&r" (roksum) \ |
102 | : "1" (addr), "Ir" (size), \ | |
103 | "r" (current_thread_info()->addr_limit) \ | |
104 | : "cc"); \ | |
105 | flag; \ | |
106 | }) | |
107 | ||
87261d19 AP |
108 | /* |
109 | * When dealing with data aborts or instruction traps we may end up with | |
110 | * a tagged userland pointer. Clear the tag to get a sane pointer to pass | |
111 | * on to access_ok(), for instance. | |
112 | */ | |
113 | #define untagged_addr(addr) sign_extend64(addr, 55) | |
114 | ||
0aea86a2 | 115 | #define access_ok(type, addr, size) __range_ok(addr, size) |
12a0ef7b | 116 | #define user_addr_max get_fs |
0aea86a2 | 117 | |
6c94f27a AB |
118 | #define _ASM_EXTABLE(from, to) \ |
119 | " .pushsection __ex_table, \"a\"\n" \ | |
120 | " .align 3\n" \ | |
121 | " .long (" #from " - .), (" #to " - .)\n" \ | |
122 | " .popsection\n" | |
123 | ||
bd38967d CM |
124 | /* |
125 | * User access enabling/disabling. | |
126 | */ | |
4b65a5db CM |
127 | #ifdef CONFIG_ARM64_SW_TTBR0_PAN |
128 | static inline void __uaccess_ttbr0_disable(void) | |
129 | { | |
130 | unsigned long ttbr; | |
131 | ||
132 | /* reserved_ttbr0 placed at the end of swapper_pg_dir */ | |
133 | ttbr = read_sysreg(ttbr1_el1) + SWAPPER_DIR_SIZE; | |
134 | write_sysreg(ttbr, ttbr0_el1); | |
135 | isb(); | |
136 | } | |
137 | ||
138 | static inline void __uaccess_ttbr0_enable(void) | |
139 | { | |
140 | unsigned long flags; | |
141 | ||
142 | /* | |
143 | * Disable interrupts to avoid preemption between reading the 'ttbr0' | |
144 | * variable and the MSR. A context switch could trigger an ASID | |
145 | * roll-over and an update of 'ttbr0'. | |
146 | */ | |
147 | local_irq_save(flags); | |
148 | write_sysreg(current_thread_info()->ttbr0, ttbr0_el1); | |
149 | isb(); | |
150 | local_irq_restore(flags); | |
151 | } | |
152 | ||
153 | static inline bool uaccess_ttbr0_disable(void) | |
154 | { | |
155 | if (!system_uses_ttbr0_pan()) | |
156 | return false; | |
157 | __uaccess_ttbr0_disable(); | |
158 | return true; | |
159 | } | |
160 | ||
161 | static inline bool uaccess_ttbr0_enable(void) | |
162 | { | |
163 | if (!system_uses_ttbr0_pan()) | |
164 | return false; | |
165 | __uaccess_ttbr0_enable(); | |
166 | return true; | |
167 | } | |
168 | #else | |
169 | static inline bool uaccess_ttbr0_disable(void) | |
170 | { | |
171 | return false; | |
172 | } | |
173 | ||
174 | static inline bool uaccess_ttbr0_enable(void) | |
175 | { | |
176 | return false; | |
177 | } | |
178 | #endif | |
179 | ||
bd38967d CM |
180 | #define __uaccess_disable(alt) \ |
181 | do { \ | |
4b65a5db CM |
182 | if (!uaccess_ttbr0_disable()) \ |
183 | asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt, \ | |
184 | CONFIG_ARM64_PAN)); \ | |
bd38967d CM |
185 | } while (0) |
186 | ||
187 | #define __uaccess_enable(alt) \ | |
188 | do { \ | |
75037120 | 189 | if (!uaccess_ttbr0_enable()) \ |
4b65a5db CM |
190 | asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt, \ |
191 | CONFIG_ARM64_PAN)); \ | |
bd38967d CM |
192 | } while (0) |
193 | ||
194 | static inline void uaccess_disable(void) | |
195 | { | |
196 | __uaccess_disable(ARM64_HAS_PAN); | |
197 | } | |
198 | ||
199 | static inline void uaccess_enable(void) | |
200 | { | |
201 | __uaccess_enable(ARM64_HAS_PAN); | |
202 | } | |
203 | ||
204 | /* | |
205 | * These functions are no-ops when UAO is present. | |
206 | */ | |
207 | static inline void uaccess_disable_not_uao(void) | |
208 | { | |
209 | __uaccess_disable(ARM64_ALT_PAN_NOT_UAO); | |
210 | } | |
211 | ||
212 | static inline void uaccess_enable_not_uao(void) | |
213 | { | |
214 | __uaccess_enable(ARM64_ALT_PAN_NOT_UAO); | |
215 | } | |
216 | ||
0aea86a2 CM |
217 | /* |
218 | * The "__xxx" versions of the user access functions do not verify the address | |
219 | * space - it must have been done previously with a separate "access_ok()" | |
220 | * call. | |
221 | * | |
222 | * The "__xxx_error" versions set the third argument to -EFAULT if an error | |
223 | * occurs, and leave it unchanged on success. | |
224 | */ | |
57f4959b | 225 | #define __get_user_asm(instr, alt_instr, reg, x, addr, err, feature) \ |
0aea86a2 | 226 | asm volatile( \ |
57f4959b JM |
227 | "1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \ |
228 | alt_instr " " reg "1, [%2]\n", feature) \ | |
0aea86a2 CM |
229 | "2:\n" \ |
230 | " .section .fixup, \"ax\"\n" \ | |
231 | " .align 2\n" \ | |
232 | "3: mov %w0, %3\n" \ | |
233 | " mov %1, #0\n" \ | |
234 | " b 2b\n" \ | |
235 | " .previous\n" \ | |
6c94f27a | 236 | _ASM_EXTABLE(1b, 3b) \ |
0aea86a2 CM |
237 | : "+r" (err), "=&r" (x) \ |
238 | : "r" (addr), "i" (-EFAULT)) | |
239 | ||
240 | #define __get_user_err(x, ptr, err) \ | |
241 | do { \ | |
242 | unsigned long __gu_val; \ | |
243 | __chk_user_ptr(ptr); \ | |
bd38967d | 244 | uaccess_enable_not_uao(); \ |
0aea86a2 CM |
245 | switch (sizeof(*(ptr))) { \ |
246 | case 1: \ | |
57f4959b JM |
247 | __get_user_asm("ldrb", "ldtrb", "%w", __gu_val, (ptr), \ |
248 | (err), ARM64_HAS_UAO); \ | |
0aea86a2 CM |
249 | break; \ |
250 | case 2: \ | |
57f4959b JM |
251 | __get_user_asm("ldrh", "ldtrh", "%w", __gu_val, (ptr), \ |
252 | (err), ARM64_HAS_UAO); \ | |
0aea86a2 CM |
253 | break; \ |
254 | case 4: \ | |
57f4959b JM |
255 | __get_user_asm("ldr", "ldtr", "%w", __gu_val, (ptr), \ |
256 | (err), ARM64_HAS_UAO); \ | |
0aea86a2 CM |
257 | break; \ |
258 | case 8: \ | |
57f4959b JM |
259 | __get_user_asm("ldr", "ldtr", "%", __gu_val, (ptr), \ |
260 | (err), ARM64_HAS_UAO); \ | |
0aea86a2 CM |
261 | break; \ |
262 | default: \ | |
263 | BUILD_BUG(); \ | |
264 | } \ | |
bd38967d | 265 | uaccess_disable_not_uao(); \ |
58fff517 | 266 | (x) = (__force __typeof__(*(ptr)))__gu_val; \ |
0aea86a2 CM |
267 | } while (0) |
268 | ||
269 | #define __get_user(x, ptr) \ | |
270 | ({ \ | |
271 | int __gu_err = 0; \ | |
272 | __get_user_err((x), (ptr), __gu_err); \ | |
273 | __gu_err; \ | |
274 | }) | |
275 | ||
276 | #define __get_user_error(x, ptr, err) \ | |
277 | ({ \ | |
278 | __get_user_err((x), (ptr), (err)); \ | |
279 | (void)0; \ | |
280 | }) | |
281 | ||
282 | #define __get_user_unaligned __get_user | |
283 | ||
284 | #define get_user(x, ptr) \ | |
285 | ({ \ | |
1f65c13e | 286 | __typeof__(*(ptr)) __user *__p = (ptr); \ |
56d2ef78 | 287 | might_fault(); \ |
1f65c13e AT |
288 | access_ok(VERIFY_READ, __p, sizeof(*__p)) ? \ |
289 | __get_user((x), __p) : \ | |
0aea86a2 CM |
290 | ((x) = 0, -EFAULT); \ |
291 | }) | |
292 | ||
57f4959b | 293 | #define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature) \ |
0aea86a2 | 294 | asm volatile( \ |
57f4959b JM |
295 | "1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \ |
296 | alt_instr " " reg "1, [%2]\n", feature) \ | |
0aea86a2 CM |
297 | "2:\n" \ |
298 | " .section .fixup,\"ax\"\n" \ | |
299 | " .align 2\n" \ | |
300 | "3: mov %w0, %3\n" \ | |
301 | " b 2b\n" \ | |
302 | " .previous\n" \ | |
6c94f27a | 303 | _ASM_EXTABLE(1b, 3b) \ |
0aea86a2 CM |
304 | : "+r" (err) \ |
305 | : "r" (x), "r" (addr), "i" (-EFAULT)) | |
306 | ||
307 | #define __put_user_err(x, ptr, err) \ | |
308 | do { \ | |
309 | __typeof__(*(ptr)) __pu_val = (x); \ | |
310 | __chk_user_ptr(ptr); \ | |
bd38967d | 311 | uaccess_enable_not_uao(); \ |
0aea86a2 CM |
312 | switch (sizeof(*(ptr))) { \ |
313 | case 1: \ | |
57f4959b JM |
314 | __put_user_asm("strb", "sttrb", "%w", __pu_val, (ptr), \ |
315 | (err), ARM64_HAS_UAO); \ | |
0aea86a2 CM |
316 | break; \ |
317 | case 2: \ | |
57f4959b JM |
318 | __put_user_asm("strh", "sttrh", "%w", __pu_val, (ptr), \ |
319 | (err), ARM64_HAS_UAO); \ | |
0aea86a2 CM |
320 | break; \ |
321 | case 4: \ | |
57f4959b JM |
322 | __put_user_asm("str", "sttr", "%w", __pu_val, (ptr), \ |
323 | (err), ARM64_HAS_UAO); \ | |
0aea86a2 CM |
324 | break; \ |
325 | case 8: \ | |
57f4959b JM |
326 | __put_user_asm("str", "sttr", "%", __pu_val, (ptr), \ |
327 | (err), ARM64_HAS_UAO); \ | |
0aea86a2 CM |
328 | break; \ |
329 | default: \ | |
330 | BUILD_BUG(); \ | |
331 | } \ | |
bd38967d | 332 | uaccess_disable_not_uao(); \ |
0aea86a2 CM |
333 | } while (0) |
334 | ||
335 | #define __put_user(x, ptr) \ | |
336 | ({ \ | |
337 | int __pu_err = 0; \ | |
338 | __put_user_err((x), (ptr), __pu_err); \ | |
339 | __pu_err; \ | |
340 | }) | |
341 | ||
342 | #define __put_user_error(x, ptr, err) \ | |
343 | ({ \ | |
344 | __put_user_err((x), (ptr), (err)); \ | |
345 | (void)0; \ | |
346 | }) | |
347 | ||
348 | #define __put_user_unaligned __put_user | |
349 | ||
350 | #define put_user(x, ptr) \ | |
351 | ({ \ | |
1f65c13e | 352 | __typeof__(*(ptr)) __user *__p = (ptr); \ |
56d2ef78 | 353 | might_fault(); \ |
1f65c13e AT |
354 | access_ok(VERIFY_WRITE, __p, sizeof(*__p)) ? \ |
355 | __put_user((x), __p) : \ | |
0aea86a2 CM |
356 | -EFAULT; \ |
357 | }) | |
358 | ||
bffe1baf YS |
359 | extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); |
360 | extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n); | |
0aea86a2 CM |
361 | extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n); |
362 | extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n); | |
363 | ||
bffe1baf YS |
364 | static inline unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n) |
365 | { | |
366 | kasan_check_write(to, n); | |
faf5b63e KC |
367 | check_object_size(to, n, false); |
368 | return __arch_copy_from_user(to, from, n); | |
bffe1baf YS |
369 | } |
370 | ||
371 | static inline unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n) | |
372 | { | |
373 | kasan_check_read(from, n); | |
faf5b63e KC |
374 | check_object_size(from, n, true); |
375 | return __arch_copy_to_user(to, from, n); | |
bffe1baf YS |
376 | } |
377 | ||
0aea86a2 CM |
378 | static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n) |
379 | { | |
4855bd25 | 380 | unsigned long res = n; |
bffe1baf | 381 | kasan_check_write(to, n); |
76624175 | 382 | check_object_size(to, n, false); |
bffe1baf | 383 | |
faf5b63e | 384 | if (access_ok(VERIFY_READ, from, n)) { |
4855bd25 AV |
385 | res = __arch_copy_from_user(to, from, n); |
386 | } | |
387 | if (unlikely(res)) | |
388 | memset(to + (n - res), 0, res); | |
389 | return res; | |
0aea86a2 CM |
390 | } |
391 | ||
392 | static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long n) | |
393 | { | |
bffe1baf | 394 | kasan_check_read(from, n); |
76624175 | 395 | check_object_size(from, n, true); |
bffe1baf | 396 | |
faf5b63e | 397 | if (access_ok(VERIFY_WRITE, to, n)) { |
bffe1baf | 398 | n = __arch_copy_to_user(to, from, n); |
faf5b63e | 399 | } |
0aea86a2 CM |
400 | return n; |
401 | } | |
402 | ||
403 | static inline unsigned long __must_check copy_in_user(void __user *to, const void __user *from, unsigned long n) | |
404 | { | |
405 | if (access_ok(VERIFY_READ, from, n) && access_ok(VERIFY_WRITE, to, n)) | |
406 | n = __copy_in_user(to, from, n); | |
407 | return n; | |
408 | } | |
409 | ||
410 | #define __copy_to_user_inatomic __copy_to_user | |
411 | #define __copy_from_user_inatomic __copy_from_user | |
412 | ||
413 | static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) | |
414 | { | |
415 | if (access_ok(VERIFY_WRITE, to, n)) | |
416 | n = __clear_user(to, n); | |
417 | return n; | |
418 | } | |
419 | ||
12a0ef7b | 420 | extern long strncpy_from_user(char *dest, const char __user *src, long count); |
0aea86a2 | 421 | |
12a0ef7b WD |
422 | extern __must_check long strlen_user(const char __user *str); |
423 | extern __must_check long strnlen_user(const char __user *str, long n); | |
0aea86a2 CM |
424 | |
425 | #endif /* __ASM_UACCESS_H */ |