2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 03, 04 by Ralf Baechle
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8 * Copyright (C) 2007 Maciej W. Rozycki
9 * Copyright (C) 2014, Imagination Technologies Ltd.
11 #ifndef _ASM_UACCESS_H
12 #define _ASM_UACCESS_H
14 #include <linux/kernel.h>
15 #include <linux/string.h>
16 #include <asm/asm-eva.h>
17 #include <asm/extable.h>
20 * The fs value determines whether argument validity checking should be
21 * performed or not. If get_fs() == USER_DS, checking is performed, with
22 * get_fs() == KERNEL_DS, checking is bypassed.
24 * For historical reasons, these macros are grossly misnamed.
28 #ifdef CONFIG_KVM_GUEST
29 #define __UA_LIMIT 0x40000000UL
31 #define __UA_LIMIT 0x80000000UL
34 #define __UA_ADDR ".word"
36 #define __UA_ADDU "addu"
40 #endif /* CONFIG_32BIT */
44 extern u64 __ua_limit;
46 #define __UA_LIMIT __ua_limit
48 #define __UA_ADDR ".dword"
50 #define __UA_ADDU "daddu"
54 #endif /* CONFIG_64BIT */
57 * USER_DS is a bitmask that has the bits set that may not be set in a valid
58 * userspace address. Note that we limit 32-bit userspace to 0x7fff8000 but
59 * the arithmetic we're doing only works if the limit is a power of two, so
60 * we use 0x80000000 here on 32-bit kernels. If a process passes an invalid
61 * address in this range it's the process's problem, not ours :-)
64 #ifdef CONFIG_KVM_GUEST
65 #define KERNEL_DS ((mm_segment_t) { 0x80000000UL })
66 #define USER_DS ((mm_segment_t) { 0xC0000000UL })
68 #define KERNEL_DS ((mm_segment_t) { 0UL })
69 #define USER_DS ((mm_segment_t) { __UA_LIMIT })
72 #define get_ds() (KERNEL_DS)
73 #define get_fs() (current_thread_info()->addr_limit)
74 #define set_fs(x) (current_thread_info()->addr_limit = (x))
76 #define segment_eq(a, b) ((a).seg == (b).seg)
79 * eva_kernel_access() - determine whether kernel memory access on an EVA system
81 * Determines whether memory accesses should be performed to kernel memory
82 * on a system using Extended Virtual Addressing (EVA).
84 * Return: true if a kernel memory access on an EVA system, else false.
86 static inline bool eva_kernel_access(void)
88 if (!IS_ENABLED(CONFIG_EVA))
91 return uaccess_kernel();
95 * Is a address valid? This does a straightforward calculation rather
99 * - "addr" doesn't have any high-bits set
100 * - AND "size" doesn't have any high-bits set
101 * - AND "addr+size" doesn't have any high-bits set
102 * - OR we are in kernel mode.
104 * __ua_size() is a trick to avoid runtime checking of positive constant
105 * sizes; for those we already know at compile time that the size is ok.
107 #define __ua_size(size) \
108 ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size))
111 * access_ok: - Checks if a user space pointer is valid
112 * @addr: User space pointer to start of block to check
113 * @size: Size of block to check
115 * Context: User context only. This function may sleep if pagefaults are
118 * Checks if a pointer to a block of memory in user space is valid.
120 * Returns true (nonzero) if the memory block may be valid, false (zero)
121 * if it is definitely invalid.
123 * Note that, depending on architecture, this function probably just
124 * checks that the pointer is in the user space range - after calling
125 * this function, memory access functions may still return -EFAULT.
128 static inline int __access_ok(const void __user *p, unsigned long size)
130 unsigned long addr = (unsigned long)p;
131 return (get_fs().seg & (addr | (addr + size) | __ua_size(size))) == 0;
134 #define access_ok(addr, size) \
135 likely(__access_ok((addr), (size)))
138 * put_user: - Write a simple value into user space.
139 * @x: Value to copy to user space.
140 * @ptr: Destination address, in user space.
142 * Context: User context only. This function may sleep if pagefaults are
145 * This macro copies a single simple value from kernel space to user
146 * space. It supports simple types like char and int, but not larger
147 * data types like structures or arrays.
149 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
150 * to the result of dereferencing @ptr.
152 * Returns zero on success, or -EFAULT on error.
154 #define put_user(x,ptr) \
155 __put_user_check((x), (ptr), sizeof(*(ptr)))
158 * get_user: - Get a simple variable from user space.
159 * @x: Variable to store result.
160 * @ptr: Source address, in user space.
162 * Context: User context only. This function may sleep if pagefaults are
165 * This macro copies a single simple variable from user space to kernel
166 * space. It supports simple types like char and int, but not larger
167 * data types like structures or arrays.
169 * @ptr must have pointer-to-simple-variable type, and the result of
170 * dereferencing @ptr must be assignable to @x without a cast.
172 * Returns zero on success, or -EFAULT on error.
173 * On error, the variable @x is set to zero.
175 #define get_user(x,ptr) \
176 __get_user_check((x), (ptr), sizeof(*(ptr)))
179 * __put_user: - Write a simple value into user space, with less checking.
180 * @x: Value to copy to user space.
181 * @ptr: Destination address, in user space.
183 * Context: User context only. This function may sleep if pagefaults are
186 * This macro copies a single simple value from kernel space to user
187 * space. It supports simple types like char and int, but not larger
188 * data types like structures or arrays.
190 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
191 * to the result of dereferencing @ptr.
193 * Caller must check the pointer with access_ok() before calling this
196 * Returns zero on success, or -EFAULT on error.
198 #define __put_user(x,ptr) \
199 __put_user_nocheck((x), (ptr), sizeof(*(ptr)))
202 * __get_user: - Get a simple variable from user space, with less checking.
203 * @x: Variable to store result.
204 * @ptr: Source address, in user space.
206 * Context: User context only. This function may sleep if pagefaults are
209 * This macro copies a single simple variable from user space to kernel
210 * space. It supports simple types like char and int, but not larger
211 * data types like structures or arrays.
213 * @ptr must have pointer-to-simple-variable type, and the result of
214 * dereferencing @ptr must be assignable to @x without a cast.
216 * Caller must check the pointer with access_ok() before calling this
219 * Returns zero on success, or -EFAULT on error.
220 * On error, the variable @x is set to zero.
222 #define __get_user(x,ptr) \
223 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
225 struct __large_struct { unsigned long buf[100]; };
226 #define __m(x) (*(struct __large_struct __user *)(x))
229 * Yuck. We need two variants, one for 64bit operation and one
230 * for 32 bit mode and old iron.
233 #define __get_kernel_common(val, size, ptr) __get_user_common(val, size, ptr)
236 * Kernel specific functions for EVA. We need to use normal load instructions
237 * to read data from kernel when operating in EVA mode. We use these macros to
238 * avoid redefining __get_user_asm for EVA.
245 #define _loadd _loadw
247 #define _loadd(reg, addr) "ld " reg ", " addr
249 #define _loadw(reg, addr) "lw " reg ", " addr
250 #define _loadh(reg, addr) "lh " reg ", " addr
251 #define _loadb(reg, addr) "lb " reg ", " addr
253 #define __get_kernel_common(val, size, ptr) \
256 case 1: __get_data_asm(val, _loadb, ptr); break; \
257 case 2: __get_data_asm(val, _loadh, ptr); break; \
258 case 4: __get_data_asm(val, _loadw, ptr); break; \
259 case 8: __GET_DW(val, _loadd, ptr); break; \
260 default: __get_user_unknown(); break; \
266 #define __GET_DW(val, insn, ptr) __get_data_asm_ll32(val, insn, ptr)
269 #define __GET_DW(val, insn, ptr) __get_data_asm(val, insn, ptr)
272 extern void __get_user_unknown(void);
274 #define __get_user_common(val, size, ptr) \
277 case 1: __get_data_asm(val, user_lb, ptr); break; \
278 case 2: __get_data_asm(val, user_lh, ptr); break; \
279 case 4: __get_data_asm(val, user_lw, ptr); break; \
280 case 8: __GET_DW(val, user_ld, ptr); break; \
281 default: __get_user_unknown(); break; \
285 #define __get_user_nocheck(x, ptr, size) \
289 if (eva_kernel_access()) { \
290 __get_kernel_common((x), size, ptr); \
292 __chk_user_ptr(ptr); \
293 __get_user_common((x), size, ptr); \
298 #define __get_user_check(x, ptr, size) \
300 int __gu_err = -EFAULT; \
301 const __typeof__(*(ptr)) __user * __gu_ptr = (ptr); \
304 if (likely(access_ok( __gu_ptr, size))) { \
305 if (eva_kernel_access()) \
306 __get_kernel_common((x), size, __gu_ptr); \
308 __get_user_common((x), size, __gu_ptr); \
315 #define __get_data_asm(val, insn, addr) \
319 __asm__ __volatile__( \
320 "1: "insn("%1", "%3")" \n" \
323 " .section .fixup,\"ax\" \n" \
328 " .section __ex_table,\"a\" \n" \
329 " "__UA_ADDR "\t1b, 3b \n" \
331 : "=r" (__gu_err), "=r" (__gu_tmp) \
332 : "0" (0), "o" (__m(addr)), "i" (-EFAULT)); \
334 (val) = (__typeof__(*(addr))) __gu_tmp; \
338 * Get a long long 64 using 32 bit registers.
340 #define __get_data_asm_ll32(val, insn, addr) \
343 unsigned long long l; \
344 __typeof__(*(addr)) t; \
347 __asm__ __volatile__( \
348 "1: " insn("%1", "(%3)")" \n" \
349 "2: " insn("%D1", "4(%3)")" \n" \
352 " .section .fixup,\"ax\" \n" \
358 " .section __ex_table,\"a\" \n" \
359 " " __UA_ADDR " 1b, 4b \n" \
360 " " __UA_ADDR " 2b, 4b \n" \
362 : "=r" (__gu_err), "=&r" (__gu_tmp.l) \
363 : "0" (0), "r" (addr), "i" (-EFAULT)); \
365 (val) = __gu_tmp.t; \
369 #define __put_kernel_common(ptr, size) __put_user_common(ptr, size)
372 * Kernel specific functions for EVA. We need to use normal load instructions
373 * to read data from kernel when operating in EVA mode. We use these macros to
374 * avoid redefining __get_data_asm for EVA.
381 #define _stored _storew
383 #define _stored(reg, addr) "ld " reg ", " addr
386 #define _storew(reg, addr) "sw " reg ", " addr
387 #define _storeh(reg, addr) "sh " reg ", " addr
388 #define _storeb(reg, addr) "sb " reg ", " addr
390 #define __put_kernel_common(ptr, size) \
393 case 1: __put_data_asm(_storeb, ptr); break; \
394 case 2: __put_data_asm(_storeh, ptr); break; \
395 case 4: __put_data_asm(_storew, ptr); break; \
396 case 8: __PUT_DW(_stored, ptr); break; \
397 default: __put_user_unknown(); break; \
403 * Yuck. We need two variants, one for 64bit operation and one
404 * for 32 bit mode and old iron.
407 #define __PUT_DW(insn, ptr) __put_data_asm_ll32(insn, ptr)
410 #define __PUT_DW(insn, ptr) __put_data_asm(insn, ptr)
413 #define __put_user_common(ptr, size) \
416 case 1: __put_data_asm(user_sb, ptr); break; \
417 case 2: __put_data_asm(user_sh, ptr); break; \
418 case 4: __put_data_asm(user_sw, ptr); break; \
419 case 8: __PUT_DW(user_sd, ptr); break; \
420 default: __put_user_unknown(); break; \
424 #define __put_user_nocheck(x, ptr, size) \
426 __typeof__(*(ptr)) __pu_val; \
430 if (eva_kernel_access()) { \
431 __put_kernel_common(ptr, size); \
433 __chk_user_ptr(ptr); \
434 __put_user_common(ptr, size); \
439 #define __put_user_check(x, ptr, size) \
441 __typeof__(*(ptr)) __user *__pu_addr = (ptr); \
442 __typeof__(*(ptr)) __pu_val = (x); \
443 int __pu_err = -EFAULT; \
446 if (likely(access_ok( __pu_addr, size))) { \
447 if (eva_kernel_access()) \
448 __put_kernel_common(__pu_addr, size); \
450 __put_user_common(__pu_addr, size); \
456 #define __put_data_asm(insn, ptr) \
458 __asm__ __volatile__( \
459 "1: "insn("%z2", "%3")" # __put_data_asm \n" \
462 " .section .fixup,\"ax\" \n" \
466 " .section __ex_table,\"a\" \n" \
467 " " __UA_ADDR " 1b, 3b \n" \
470 : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)), \
474 #define __put_data_asm_ll32(insn, ptr) \
476 __asm__ __volatile__( \
477 "1: "insn("%2", "(%3)")" # __put_data_asm_ll32 \n" \
478 "2: "insn("%D2", "4(%3)")" \n" \
481 " .section .fixup,\"ax\" \n" \
485 " .section __ex_table,\"a\" \n" \
486 " " __UA_ADDR " 1b, 4b \n" \
487 " " __UA_ADDR " 2b, 4b \n" \
490 : "0" (0), "r" (__pu_val), "r" (ptr), \
494 extern void __put_user_unknown(void);
497 * We're generating jump to subroutines which will be outside the range of
501 #define __MODULE_JAL(destination) \
503 __UA_LA "\t$1, " #destination "\n\t" \
507 #define __MODULE_JAL(destination) \
508 "jal\t" #destination "\n\t"
511 #if defined(CONFIG_CPU_DADDI_WORKAROUNDS) || (defined(CONFIG_EVA) && \
512 defined(CONFIG_CPU_HAS_PREFETCH))
513 #define DADDI_SCRATCH "$3"
515 #define DADDI_SCRATCH "$0"
518 extern size_t __copy_user(void *__to, const void *__from, size_t __n);
520 #define __invoke_copy_from(func, to, from, n) \
522 register void *__cu_to_r __asm__("$4"); \
523 register const void __user *__cu_from_r __asm__("$5"); \
524 register long __cu_len_r __asm__("$6"); \
527 __cu_from_r = (from); \
529 __asm__ __volatile__( \
530 ".set\tnoreorder\n\t" \
533 __UA_ADDU "\t$1, %1, %2\n\t" \
536 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
538 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
539 DADDI_SCRATCH, "memory"); \
543 #define __invoke_copy_to(func, to, from, n) \
545 register void __user *__cu_to_r __asm__("$4"); \
546 register const void *__cu_from_r __asm__("$5"); \
547 register long __cu_len_r __asm__("$6"); \
550 __cu_from_r = (from); \
552 __asm__ __volatile__( \
554 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
556 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
557 DADDI_SCRATCH, "memory"); \
561 #define __invoke_copy_from_kernel(to, from, n) \
562 __invoke_copy_from(__copy_user, to, from, n)
564 #define __invoke_copy_to_kernel(to, from, n) \
565 __invoke_copy_to(__copy_user, to, from, n)
567 #define ___invoke_copy_in_kernel(to, from, n) \
568 __invoke_copy_from(__copy_user, to, from, n)
571 #define __invoke_copy_from_user(to, from, n) \
572 __invoke_copy_from(__copy_user, to, from, n)
574 #define __invoke_copy_to_user(to, from, n) \
575 __invoke_copy_to(__copy_user, to, from, n)
577 #define ___invoke_copy_in_user(to, from, n) \
578 __invoke_copy_from(__copy_user, to, from, n)
582 /* EVA specific functions */
584 extern size_t __copy_from_user_eva(void *__to, const void *__from,
586 extern size_t __copy_to_user_eva(void *__to, const void *__from,
588 extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n);
591 * Source or destination address is in userland. We need to go through
594 #define __invoke_copy_from_user(to, from, n) \
595 __invoke_copy_from(__copy_from_user_eva, to, from, n)
597 #define __invoke_copy_to_user(to, from, n) \
598 __invoke_copy_to(__copy_to_user_eva, to, from, n)
600 #define ___invoke_copy_in_user(to, from, n) \
601 __invoke_copy_from(__copy_in_user_eva, to, from, n)
603 #endif /* CONFIG_EVA */
605 static inline unsigned long
606 raw_copy_to_user(void __user *to, const void *from, unsigned long n)
608 if (eva_kernel_access())
609 return __invoke_copy_to_kernel(to, from, n);
611 return __invoke_copy_to_user(to, from, n);
614 static inline unsigned long
615 raw_copy_from_user(void *to, const void __user *from, unsigned long n)
617 if (eva_kernel_access())
618 return __invoke_copy_from_kernel(to, from, n);
620 return __invoke_copy_from_user(to, from, n);
623 #define INLINE_COPY_FROM_USER
624 #define INLINE_COPY_TO_USER
626 static inline unsigned long
627 raw_copy_in_user(void __user*to, const void __user *from, unsigned long n)
629 if (eva_kernel_access())
630 return ___invoke_copy_in_kernel(to, from, n);
632 return ___invoke_copy_in_user(to, from, n);
635 extern __kernel_size_t __bzero_kernel(void __user *addr, __kernel_size_t size);
636 extern __kernel_size_t __bzero(void __user *addr, __kernel_size_t size);
639 * __clear_user: - Zero a block of memory in user space, with less checking.
640 * @to: Destination address, in user space.
641 * @n: Number of bytes to zero.
643 * Zero a block of memory in user space. Caller must check
644 * the specified block with access_ok() before calling this function.
646 * Returns number of bytes that could not be cleared.
647 * On success, this will be zero.
649 static inline __kernel_size_t
650 __clear_user(void __user *addr, __kernel_size_t size)
654 #ifdef CONFIG_CPU_MICROMIPS
655 /* micromips memset / bzero also clobbers t7 & t8 */
656 #define bzero_clobbers "$4", "$5", "$6", __UA_t0, __UA_t1, "$15", "$24", "$31"
658 #define bzero_clobbers "$4", "$5", "$6", __UA_t0, __UA_t1, "$31"
659 #endif /* CONFIG_CPU_MICROMIPS */
661 if (eva_kernel_access()) {
662 __asm__ __volatile__(
666 __MODULE_JAL(__bzero_kernel)
669 : "r" (addr), "r" (size)
673 __asm__ __volatile__(
677 __MODULE_JAL(__bzero)
680 : "r" (addr), "r" (size)
687 #define clear_user(addr,n) \
689 void __user * __cl_addr = (addr); \
690 unsigned long __cl_size = (n); \
691 if (__cl_size && access_ok(__cl_addr, __cl_size)) \
692 __cl_size = __clear_user(__cl_addr, __cl_size); \
696 extern long __strncpy_from_kernel_asm(char *__to, const char __user *__from, long __len);
697 extern long __strncpy_from_user_asm(char *__to, const char __user *__from, long __len);
700 * strncpy_from_user: - Copy a NUL terminated string from userspace.
701 * @dst: Destination address, in kernel space. This buffer must be at
702 * least @count bytes long.
703 * @src: Source address, in user space.
704 * @count: Maximum number of bytes to copy, including the trailing NUL.
706 * Copies a NUL-terminated string from userspace to kernel space.
708 * On success, returns the length of the string (not including the trailing
711 * If access to userspace fails, returns -EFAULT (some data may have been
714 * If @count is smaller than the length of the string, copies @count bytes
715 * and returns @count.
718 strncpy_from_user(char *__to, const char __user *__from, long __len)
722 if (eva_kernel_access()) {
723 __asm__ __volatile__(
727 __MODULE_JAL(__strncpy_from_kernel_asm)
730 : "r" (__to), "r" (__from), "r" (__len)
731 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
734 __asm__ __volatile__(
738 __MODULE_JAL(__strncpy_from_user_asm)
741 : "r" (__to), "r" (__from), "r" (__len)
742 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
748 extern long __strnlen_kernel_asm(const char __user *s, long n);
749 extern long __strnlen_user_asm(const char __user *s, long n);
752 * strnlen_user: - Get the size of a string in user space.
753 * @str: The string to measure.
755 * Context: User context only. This function may sleep if pagefaults are
758 * Get the size of a NUL-terminated string in user space.
760 * Returns the size of the string INCLUDING the terminating NUL.
761 * On exception, returns 0.
762 * If the string is too long, returns a value greater than @n.
764 static inline long strnlen_user(const char __user *s, long n)
769 if (eva_kernel_access()) {
770 __asm__ __volatile__(
773 __MODULE_JAL(__strnlen_kernel_asm)
777 : "$2", "$4", "$5", __UA_t0, "$31");
779 __asm__ __volatile__(
782 __MODULE_JAL(__strnlen_user_asm)
786 : "$2", "$4", "$5", __UA_t0, "$31");
792 #endif /* _ASM_UACCESS_H */