[linux-2.6-block.git] / arch / x86 / lib / usercopy_32.c

// SPDX-License-Identifier: GPL-2.0
/*
 * User address space access functions.
 * The non inlined parts of asm-i386/uaccess.h are here.
 *
 * Copyright 1997 Andi Kleen <ak@muc.de>
 * Copyright 1997 Linus Torvalds
 */
#include <linux/export.h>
#include <linux/uaccess.h>
#include <asm/mmx.h>
#include <asm/asm.h>

#ifdef CONFIG_X86_INTEL_USERCOPY
/*
 * Alignment at which movsl is preferred for bulk memory copies.
 */
struct movsl_mask movsl_mask __read_mostly;
#endif

static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n)
{
#ifdef CONFIG_X86_INTEL_USERCOPY
	if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask))
		return 0;
#endif
	return 1;
}
#define movsl_is_ok(a1, a2, n) \
	__movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n))

/*
 * Zero Userspace
 */

#define __do_clear_user(addr,size)					\
do {									\
	int __d0;							\
	might_fault();							\
	__asm__ __volatile__(						\
		ASM_STAC "\n"						\
		"0:	rep; stosl\n"					\
		"	movl %2,%0\n"					\
		"1:	rep; stosb\n"					\
		"2: " ASM_CLAC "\n"					\
		".section .fixup,\"ax\"\n"				\
		"3:	lea 0(%2,%0,4),%0\n"				\
		"	jmp 2b\n"					\
		".previous\n"						\
		_ASM_EXTABLE_UA(0b, 3b)					\
		_ASM_EXTABLE_UA(1b, 2b)					\
		: "=&c"(size), "=&D" (__d0)				\
		: "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0));	\
} while (0)

/**
 * clear_user: - Zero a block of memory in user space.
 * @to:   Destination address, in user space.
 * @n:    Number of bytes to zero.
 *
 * Zero a block of memory in user space.
 *
 * Returns number of bytes that could not be cleared.
 * On success, this will be zero.
 */
unsigned long
clear_user(void __user *to, unsigned long n)
{
	might_fault();
	if (access_ok(to, n))
		__do_clear_user(to, n);
	return n;
}
EXPORT_SYMBOL(clear_user);

/**
 * __clear_user: - Zero a block of memory in user space, with less checking.
 * @to:   Destination address, in user space.
 * @n:    Number of bytes to zero.
 *
 * Zero a block of memory in user space.  Caller must check
 * the specified block with access_ok() before calling this function.
 *
 * Returns number of bytes that could not be cleared.
 * On success, this will be zero.
 */
unsigned long
__clear_user(void __user *to, unsigned long n)
{
	__do_clear_user(to, n);
	return n;
}
EXPORT_SYMBOL(__clear_user);

#ifdef CONFIG_X86_INTEL_USERCOPY
static unsigned long
__copy_user_intel(void __user *to, const void *from, unsigned long size)
{
	int d0, d1;
	__asm__ __volatile__(
		       "       .align 2,0x90\n"
		       "1:     movl 32(%4), %%eax\n"
		       "       cmpl $67, %0\n"
		       "       jbe 3f\n"
		       "2:     movl 64(%4), %%eax\n"
		       "       .align 2,0x90\n"
		       "3:     movl 0(%4), %%eax\n"
		       "4:     movl 4(%4), %%edx\n"
		       "5:     movl %%eax, 0(%3)\n"
		       "6:     movl %%edx, 4(%3)\n"
		       "7:     movl 8(%4), %%eax\n"
		       "8:     movl 12(%4),%%edx\n"
		       "9:     movl %%eax, 8(%3)\n"
		       "10:    movl %%edx, 12(%3)\n"
		       "11:    movl 16(%4), %%eax\n"
		       "12:    movl 20(%4), %%edx\n"
		       "13:    movl %%eax, 16(%3)\n"
		       "14:    movl %%edx, 20(%3)\n"
		       "15:    movl 24(%4), %%eax\n"
		       "16:    movl 28(%4), %%edx\n"
		       "17:    movl %%eax, 24(%3)\n"
		       "18:    movl %%edx, 28(%3)\n"
		       "19:    movl 32(%4), %%eax\n"
		       "20:    movl 36(%4), %%edx\n"
		       "21:    movl %%eax, 32(%3)\n"
		       "22:    movl %%edx, 36(%3)\n"
		       "23:    movl 40(%4), %%eax\n"
		       "24:    movl 44(%4), %%edx\n"
		       "25:    movl %%eax, 40(%3)\n"
		       "26:    movl %%edx, 44(%3)\n"
		       "27:    movl 48(%4), %%eax\n"
		       "28:    movl 52(%4), %%edx\n"
		       "29:    movl %%eax, 48(%3)\n"
		       "30:    movl %%edx, 52(%3)\n"
		       "31:    movl 56(%4), %%eax\n"
		       "32:    movl 60(%4), %%edx\n"
		       "33:    movl %%eax, 56(%3)\n"
		       "34:    movl %%edx, 60(%3)\n"
		       "       addl $-64, %0\n"
		       "       addl $64, %4\n"
		       "       addl $64, %3\n"
		       "       cmpl $63, %0\n"
		       "       ja  1b\n"
		       "35:    movl  %0, %%eax\n"
		       "       shrl  $2, %0\n"
		       "       andl  $3, %%eax\n"
		       "       cld\n"
		       "99:    rep; movsl\n"
		       "36:    movl %%eax, %0\n"
		       "37:    rep; movsb\n"
		       "100:\n"
		       ".section .fixup,\"ax\"\n"
		       "101:   lea 0(%%eax,%0,4),%0\n"
		       "       jmp 100b\n"
		       ".previous\n"
		       _ASM_EXTABLE_UA(1b, 100b)
		       _ASM_EXTABLE_UA(2b, 100b)
		       _ASM_EXTABLE_UA(3b, 100b)
		       _ASM_EXTABLE_UA(4b, 100b)
		       _ASM_EXTABLE_UA(5b, 100b)
		       _ASM_EXTABLE_UA(6b, 100b)
		       _ASM_EXTABLE_UA(7b, 100b)
		       _ASM_EXTABLE_UA(8b, 100b)
		       _ASM_EXTABLE_UA(9b, 100b)
		       _ASM_EXTABLE_UA(10b, 100b)
		       _ASM_EXTABLE_UA(11b, 100b)
		       _ASM_EXTABLE_UA(12b, 100b)
		       _ASM_EXTABLE_UA(13b, 100b)
		       _ASM_EXTABLE_UA(14b, 100b)
		       _ASM_EXTABLE_UA(15b, 100b)
		       _ASM_EXTABLE_UA(16b, 100b)
		       _ASM_EXTABLE_UA(17b, 100b)
		       _ASM_EXTABLE_UA(18b, 100b)
		       _ASM_EXTABLE_UA(19b, 100b)
		       _ASM_EXTABLE_UA(20b, 100b)
		       _ASM_EXTABLE_UA(21b, 100b)
		       _ASM_EXTABLE_UA(22b, 100b)
		       _ASM_EXTABLE_UA(23b, 100b)
		       _ASM_EXTABLE_UA(24b, 100b)
		       _ASM_EXTABLE_UA(25b, 100b)
		       _ASM_EXTABLE_UA(26b, 100b)
		       _ASM_EXTABLE_UA(27b, 100b)
		       _ASM_EXTABLE_UA(28b, 100b)
		       _ASM_EXTABLE_UA(29b, 100b)
		       _ASM_EXTABLE_UA(30b, 100b)
		       _ASM_EXTABLE_UA(31b, 100b)
		       _ASM_EXTABLE_UA(32b, 100b)
		       _ASM_EXTABLE_UA(33b, 100b)
		       _ASM_EXTABLE_UA(34b, 100b)
		       _ASM_EXTABLE_UA(35b, 100b)
		       _ASM_EXTABLE_UA(36b, 100b)
		       _ASM_EXTABLE_UA(37b, 100b)
		       _ASM_EXTABLE_UA(99b, 101b)
		       : "=&c"(size), "=&D" (d0), "=&S" (d1)
		       :  "1"(to), "2"(from), "0"(size)
		       : "eax", "edx", "memory");
	return size;
}

static unsigned long __copy_user_intel_nocache(void *to,
				const void __user *from, unsigned long size)
{
	int d0, d1;

	__asm__ __volatile__(
	       "        .align 2,0x90\n"
	       "0:      movl 32(%4), %%eax\n"
	       "        cmpl $67, %0\n"
	       "        jbe 2f\n"
	       "1:      movl 64(%4), %%eax\n"
	       "        .align 2,0x90\n"
	       "2:      movl 0(%4), %%eax\n"
	       "21:     movl 4(%4), %%edx\n"
	       "        movnti %%eax, 0(%3)\n"
	       "        movnti %%edx, 4(%3)\n"
	       "3:      movl 8(%4), %%eax\n"
	       "31:     movl 12(%4),%%edx\n"
	       "        movnti %%eax, 8(%3)\n"
	       "        movnti %%edx, 12(%3)\n"
	       "4:      movl 16(%4), %%eax\n"
	       "41:     movl 20(%4), %%edx\n"
	       "        movnti %%eax, 16(%3)\n"
	       "        movnti %%edx, 20(%3)\n"
	       "10:     movl 24(%4), %%eax\n"
	       "51:     movl 28(%4), %%edx\n"
	       "        movnti %%eax, 24(%3)\n"
	       "        movnti %%edx, 28(%3)\n"
	       "11:     movl 32(%4), %%eax\n"
	       "61:     movl 36(%4), %%edx\n"
	       "        movnti %%eax, 32(%3)\n"
	       "        movnti %%edx, 36(%3)\n"
	       "12:     movl 40(%4), %%eax\n"
	       "71:     movl 44(%4), %%edx\n"
	       "        movnti %%eax, 40(%3)\n"
	       "        movnti %%edx, 44(%3)\n"
	       "13:     movl 48(%4), %%eax\n"
	       "81:     movl 52(%4), %%edx\n"
	       "        movnti %%eax, 48(%3)\n"
	       "        movnti %%edx, 52(%3)\n"
	       "14:     movl 56(%4), %%eax\n"
	       "91:     movl 60(%4), %%edx\n"
	       "        movnti %%eax, 56(%3)\n"
	       "        movnti %%edx, 60(%3)\n"
	       "        addl $-64, %0\n"
	       "        addl $64, %4\n"
	       "        addl $64, %3\n"
	       "        cmpl $63, %0\n"
	       "        ja  0b\n"
	       "        sfence \n"
	       "5:      movl  %0, %%eax\n"
	       "        shrl  $2, %0\n"
	       "        andl $3, %%eax\n"
	       "        cld\n"
	       "6:      rep; movsl\n"
	       "        movl %%eax,%0\n"
	       "7:      rep; movsb\n"
	       "8:\n"
	       ".section .fixup,\"ax\"\n"
	       "9:      lea 0(%%eax,%0,4),%0\n"
	       "16:     jmp 8b\n"
	       ".previous\n"
	       _ASM_EXTABLE_UA(0b, 16b)
	       _ASM_EXTABLE_UA(1b, 16b)
	       _ASM_EXTABLE_UA(2b, 16b)
	       _ASM_EXTABLE_UA(21b, 16b)
	       _ASM_EXTABLE_UA(3b, 16b)
	       _ASM_EXTABLE_UA(31b, 16b)
	       _ASM_EXTABLE_UA(4b, 16b)
	       _ASM_EXTABLE_UA(41b, 16b)
	       _ASM_EXTABLE_UA(10b, 16b)
	       _ASM_EXTABLE_UA(51b, 16b)
	       _ASM_EXTABLE_UA(11b, 16b)
	       _ASM_EXTABLE_UA(61b, 16b)
	       _ASM_EXTABLE_UA(12b, 16b)
	       _ASM_EXTABLE_UA(71b, 16b)
	       _ASM_EXTABLE_UA(13b, 16b)
	       _ASM_EXTABLE_UA(81b, 16b)
	       _ASM_EXTABLE_UA(14b, 16b)
	       _ASM_EXTABLE_UA(91b, 16b)
	       _ASM_EXTABLE_UA(6b, 9b)
	       _ASM_EXTABLE_UA(7b, 16b)
	       : "=&c"(size), "=&D" (d0), "=&S" (d1)
	       :  "1"(to), "2"(from), "0"(size)
	       : "eax", "edx", "memory");
	return size;
}

#else

/*
 * Leave these declared but undefined.  They should not be any references to
 * them
 */
unsigned long __copy_user_intel(void __user *to, const void *from,
					unsigned long size);
#endif /* CONFIG_X86_INTEL_USERCOPY */

/* Generic arbitrary sized copy.  */
#define __copy_user(to, from, size)					\
do {									\
	int __d0, __d1, __d2;						\
	__asm__ __volatile__(						\
		"	cmp  $7,%0\n"					\
		"	jbe  1f\n"					\
		"	movl %1,%0\n"					\
		"	negl %0\n"					\
		"	andl $7,%0\n"					\
		"	subl %0,%3\n"					\
		"4:	rep; movsb\n"					\
		"	movl %3,%0\n"					\
		"	shrl $2,%0\n"					\
		"	andl $3,%3\n"					\
		"	.align 2,0x90\n"				\
		"0:	rep; movsl\n"					\
		"	movl %3,%0\n"					\
		"1:	rep; movsb\n"					\
		"2:\n"							\
		".section .fixup,\"ax\"\n"				\
		"5:	addl %3,%0\n"					\
		"	jmp 2b\n"					\
		"3:	lea 0(%3,%0,4),%0\n"				\
		"	jmp 2b\n"					\
		".previous\n"						\
		_ASM_EXTABLE_UA(4b, 5b)					\
		_ASM_EXTABLE_UA(0b, 3b)					\
		_ASM_EXTABLE_UA(1b, 2b)					\
		: "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)	\
		: "3"(size), "0"(size), "1"(to), "2"(from)		\
		: "memory");						\
} while (0)

unsigned long __copy_user_ll(void *to, const void *from, unsigned long n)
{
	__uaccess_begin_nospec();
	if (movsl_is_ok(to, from, n))
		__copy_user(to, from, n);
	else
		n = __copy_user_intel(to, from, n);
	__uaccess_end();
	return n;
}
EXPORT_SYMBOL(__copy_user_ll);

unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
					unsigned long n)
{
	__uaccess_begin_nospec();
#ifdef CONFIG_X86_INTEL_USERCOPY
	if (n > 64 && static_cpu_has(X86_FEATURE_XMM2))
		n = __copy_user_intel_nocache(to, from, n);
	else
		__copy_user(to, from, n);
#else
	__copy_user(to, from, n);
#endif
	__uaccess_end();
	return n;
}
EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
3f50dbc1	2	/*
1da177e4 LT	3	* User address space access functions.
	4	* The non inlined parts of asm-i386/uaccess.h are here.
	5	*
	6	* Copyright 1997 Andi Kleen <ak@muc.de>
	7	* Copyright 1997 Linus Torvalds
	8	*/
e683014c	9	#include <linux/export.h>
7c0f6ba6	10	#include <linux/uaccess.h>
1da177e4	11	#include <asm/mmx.h>
9c675128	12	#include <asm/asm.h>
1da177e4	13
8bfcb396 TP	14	#ifdef CONFIG_X86_INTEL_USERCOPY
	15	/*
	16	* Alignment at which movsl is preferred for bulk memory copies.
	17	*/
	18	struct movsl_mask movsl_mask __read_mostly;
	19	#endif
	20
1da177e4 LT	21	static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n)
	22	{
	23	#ifdef CONFIG_X86_INTEL_USERCOPY
	24	if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask))
	25	return 0;
	26	#endif
	27	return 1;
	28	}
3f50dbc1 PC	29	#define movsl_is_ok(a1, a2, n) \
3f50dbc1 PC	30	__movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n))
1da177e4	31
1da177e4 LT	32	/*
	33	* Zero Userspace
	34	*/
	35
	36	#define __do_clear_user(addr,size) \
	37	do { \
	38	int __d0; \
3ee1afa3	39	might_fault(); \
3f50dbc1	40	__asm__ __volatile__( \
63bcff2a	41	ASM_STAC "\n" \
1da177e4 LT	42	"0: rep; stosl\n" \
	43	" movl %2,%0\n" \
	44	"1: rep; stosb\n" \
63bcff2a	45	"2: " ASM_CLAC "\n" \
1da177e4 LT	46	".section .fixup,\"ax\"\n" \
	47	"3: lea 0(%2,%0,4),%0\n" \
	48	" jmp 2b\n" \
	49	".previous\n" \
75045f77 JH	50	_ASM_EXTABLE_UA(0b, 3b) \
75045f77 JH	51	_ASM_EXTABLE_UA(1b, 2b) \
1da177e4 LT	52	: "=&c"(size), "=&D" (__d0) \
	53	: "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \
	54	} while (0)
	55
	56	/**
	57	* clear_user: - Zero a block of memory in user space.
	58	* @to: Destination address, in user space.
	59	* @n: Number of bytes to zero.
	60	*
	61	* Zero a block of memory in user space.
	62	*
	63	* Returns number of bytes that could not be cleared.
	64	* On success, this will be zero.
	65	*/
	66	unsigned long
	67	clear_user(void __user *to, unsigned long n)
	68	{
1d18ef48	69	might_fault();
96d4f267	70	if (access_ok(to, n))
1da177e4 LT	71	__do_clear_user(to, n);
	72	return n;
	73	}
129f6946	74	EXPORT_SYMBOL(clear_user);
1da177e4 LT	75
	76	/**
	77	* __clear_user: - Zero a block of memory in user space, with less checking.
	78	* @to: Destination address, in user space.
	79	* @n: Number of bytes to zero.
	80	*
	81	* Zero a block of memory in user space. Caller must check
	82	* the specified block with access_ok() before calling this function.
	83	*
	84	* Returns number of bytes that could not be cleared.
	85	* On success, this will be zero.
	86	*/
	87	unsigned long
	88	__clear_user(void __user *to, unsigned long n)
	89	{
	90	__do_clear_user(to, n);
	91	return n;
	92	}
129f6946	93	EXPORT_SYMBOL(__clear_user);
1da177e4	94
1da177e4 LT	95	#ifdef CONFIG_X86_INTEL_USERCOPY
	96	static unsigned long
	97	__copy_user_intel(void __user to, const void from, unsigned long size)
	98	{
	99	int d0, d1;
	100	__asm__ __volatile__(
	101	" .align 2,0x90\n"
	102	"1: movl 32(%4), %%eax\n"
	103	" cmpl $67, %0\n"
	104	" jbe 3f\n"
	105	"2: movl 64(%4), %%eax\n"
	106	" .align 2,0x90\n"
	107	"3: movl 0(%4), %%eax\n"
	108	"4: movl 4(%4), %%edx\n"
	109	"5: movl %%eax, 0(%3)\n"
	110	"6: movl %%edx, 4(%3)\n"
	111	"7: movl 8(%4), %%eax\n"
	112	"8: movl 12(%4),%%edx\n"
	113	"9: movl %%eax, 8(%3)\n"
	114	"10: movl %%edx, 12(%3)\n"
	115	"11: movl 16(%4), %%eax\n"
	116	"12: movl 20(%4), %%edx\n"
	117	"13: movl %%eax, 16(%3)\n"
	118	"14: movl %%edx, 20(%3)\n"
	119	"15: movl 24(%4), %%eax\n"
	120	"16: movl 28(%4), %%edx\n"
	121	"17: movl %%eax, 24(%3)\n"
	122	"18: movl %%edx, 28(%3)\n"
	123	"19: movl 32(%4), %%eax\n"
	124	"20: movl 36(%4), %%edx\n"
	125	"21: movl %%eax, 32(%3)\n"
	126	"22: movl %%edx, 36(%3)\n"
	127	"23: movl 40(%4), %%eax\n"
	128	"24: movl 44(%4), %%edx\n"
	129	"25: movl %%eax, 40(%3)\n"
	130	"26: movl %%edx, 44(%3)\n"
	131	"27: movl 48(%4), %%eax\n"
	132	"28: movl 52(%4), %%edx\n"
	133	"29: movl %%eax, 48(%3)\n"
	134	"30: movl %%edx, 52(%3)\n"
	135	"31: movl 56(%4), %%eax\n"
	136	"32: movl 60(%4), %%edx\n"
	137	"33: movl %%eax, 56(%3)\n"
	138	"34: movl %%edx, 60(%3)\n"
	139	" addl $-64, %0\n"
	140	" addl $64, %4\n"
	141	" addl $64, %3\n"
	142	" cmpl $63, %0\n"
	143	" ja 1b\n"
	144	"35: movl %0, %%eax\n"
	145	" shrl $2, %0\n"
	146	" andl $3, %%eax\n"
	147	" cld\n"
	148	"99: rep; movsl\n"
	149	"36: movl %%eax, %0\n"
	150	"37: rep; movsb\n"
	151	"100:\n"
	152	".section .fixup,\"ax\"\n"
	153	"101: lea 0(%%eax,%0,4),%0\n"
	154	" jmp 100b\n"
	155	".previous\n"
75045f77 JH	156	_ASM_EXTABLE_UA(1b, 100b)
	157	_ASM_EXTABLE_UA(2b, 100b)
	158	_ASM_EXTABLE_UA(3b, 100b)
	159	_ASM_EXTABLE_UA(4b, 100b)
	160	_ASM_EXTABLE_UA(5b, 100b)
	161	_ASM_EXTABLE_UA(6b, 100b)
	162	_ASM_EXTABLE_UA(7b, 100b)
	163	_ASM_EXTABLE_UA(8b, 100b)
	164	_ASM_EXTABLE_UA(9b, 100b)
	165	_ASM_EXTABLE_UA(10b, 100b)
	166	_ASM_EXTABLE_UA(11b, 100b)
	167	_ASM_EXTABLE_UA(12b, 100b)
	168	_ASM_EXTABLE_UA(13b, 100b)
	169	_ASM_EXTABLE_UA(14b, 100b)
	170	_ASM_EXTABLE_UA(15b, 100b)
	171	_ASM_EXTABLE_UA(16b, 100b)
	172	_ASM_EXTABLE_UA(17b, 100b)
	173	_ASM_EXTABLE_UA(18b, 100b)
	174	_ASM_EXTABLE_UA(19b, 100b)
	175	_ASM_EXTABLE_UA(20b, 100b)
	176	_ASM_EXTABLE_UA(21b, 100b)
	177	_ASM_EXTABLE_UA(22b, 100b)
	178	_ASM_EXTABLE_UA(23b, 100b)
	179	_ASM_EXTABLE_UA(24b, 100b)
	180	_ASM_EXTABLE_UA(25b, 100b)
	181	_ASM_EXTABLE_UA(26b, 100b)
	182	_ASM_EXTABLE_UA(27b, 100b)
	183	_ASM_EXTABLE_UA(28b, 100b)
	184	_ASM_EXTABLE_UA(29b, 100b)
	185	_ASM_EXTABLE_UA(30b, 100b)
	186	_ASM_EXTABLE_UA(31b, 100b)
	187	_ASM_EXTABLE_UA(32b, 100b)
	188	_ASM_EXTABLE_UA(33b, 100b)
	189	_ASM_EXTABLE_UA(34b, 100b)
	190	_ASM_EXTABLE_UA(35b, 100b)
	191	_ASM_EXTABLE_UA(36b, 100b)
	192	_ASM_EXTABLE_UA(37b, 100b)
	193	_ASM_EXTABLE_UA(99b, 101b)
1da177e4 LT	194	: "=&c"(size), "=&D" (d0), "=&S" (d1)
	195	: "1"(to), "2"(from), "0"(size)
	196	: "eax", "edx", "memory");
	197	return size;
	198	}
	199
7c12d811 N	200	static unsigned long __copy_user_intel_nocache(void *to,
	201	const void __user *from, unsigned long size)
	202	{
3f50dbc1	203	int d0, d1;
7c12d811 N	204
	205	__asm__ __volatile__(
	206	" .align 2,0x90\n"
	207	"0: movl 32(%4), %%eax\n"
	208	" cmpl $67, %0\n"
	209	" jbe 2f\n"
	210	"1: movl 64(%4), %%eax\n"
	211	" .align 2,0x90\n"
	212	"2: movl 0(%4), %%eax\n"
	213	"21: movl 4(%4), %%edx\n"
	214	" movnti %%eax, 0(%3)\n"
	215	" movnti %%edx, 4(%3)\n"
	216	"3: movl 8(%4), %%eax\n"
	217	"31: movl 12(%4),%%edx\n"
	218	" movnti %%eax, 8(%3)\n"
	219	" movnti %%edx, 12(%3)\n"
	220	"4: movl 16(%4), %%eax\n"
	221	"41: movl 20(%4), %%edx\n"
	222	" movnti %%eax, 16(%3)\n"
	223	" movnti %%edx, 20(%3)\n"
	224	"10: movl 24(%4), %%eax\n"
	225	"51: movl 28(%4), %%edx\n"
	226	" movnti %%eax, 24(%3)\n"
	227	" movnti %%edx, 28(%3)\n"
	228	"11: movl 32(%4), %%eax\n"
	229	"61: movl 36(%4), %%edx\n"
	230	" movnti %%eax, 32(%3)\n"
	231	" movnti %%edx, 36(%3)\n"
	232	"12: movl 40(%4), %%eax\n"
	233	"71: movl 44(%4), %%edx\n"
	234	" movnti %%eax, 40(%3)\n"
	235	" movnti %%edx, 44(%3)\n"
	236	"13: movl 48(%4), %%eax\n"
	237	"81: movl 52(%4), %%edx\n"
	238	" movnti %%eax, 48(%3)\n"
	239	" movnti %%edx, 52(%3)\n"
	240	"14: movl 56(%4), %%eax\n"
	241	"91: movl 60(%4), %%edx\n"
	242	" movnti %%eax, 56(%3)\n"
	243	" movnti %%edx, 60(%3)\n"
	244	" addl $-64, %0\n"
	245	" addl $64, %4\n"
	246	" addl $64, %3\n"
	247	" cmpl $63, %0\n"
	248	" ja 0b\n"
	249	" sfence \n"
	250	"5: movl %0, %%eax\n"
	251	" shrl $2, %0\n"
	252	" andl $3, %%eax\n"
	253	" cld\n"
	254	"6: rep; movsl\n"
	255	" movl %%eax,%0\n"
	256	"7: rep; movsb\n"
	257	"8:\n"
	258	".section .fixup,\"ax\"\n"
	259	"9: lea 0(%%eax,%0,4),%0\n"
	260	"16: jmp 8b\n"
	261	".previous\n"
75045f77 JH	262	_ASM_EXTABLE_UA(0b, 16b)
	263	_ASM_EXTABLE_UA(1b, 16b)
	264	_ASM_EXTABLE_UA(2b, 16b)
	265	_ASM_EXTABLE_UA(21b, 16b)
	266	_ASM_EXTABLE_UA(3b, 16b)
	267	_ASM_EXTABLE_UA(31b, 16b)
	268	_ASM_EXTABLE_UA(4b, 16b)
	269	_ASM_EXTABLE_UA(41b, 16b)
	270	_ASM_EXTABLE_UA(10b, 16b)
	271	_ASM_EXTABLE_UA(51b, 16b)
	272	_ASM_EXTABLE_UA(11b, 16b)
	273	_ASM_EXTABLE_UA(61b, 16b)
	274	_ASM_EXTABLE_UA(12b, 16b)
	275	_ASM_EXTABLE_UA(71b, 16b)
	276	_ASM_EXTABLE_UA(13b, 16b)
	277	_ASM_EXTABLE_UA(81b, 16b)
	278	_ASM_EXTABLE_UA(14b, 16b)
	279	_ASM_EXTABLE_UA(91b, 16b)
	280	_ASM_EXTABLE_UA(6b, 9b)
	281	_ASM_EXTABLE_UA(7b, 16b)
7c12d811 N	282	: "=&c"(size), "=&D" (d0), "=&S" (d1)
	283	: "1"(to), "2"(from), "0"(size)
	284	: "eax", "edx", "memory");
	285	return size;
	286	}
	287
1da177e4	288	#else
c22ce143	289
1da177e4 LT	290	/*
	291	* Leave these declared but undefined. They should not be any references to
	292	* them
	293	*/
c22ce143 HY	294	unsigned long __copy_user_intel(void __user to, const void from,
c22ce143 HY	295	unsigned long size);
1da177e4 LT	296	#endif /* CONFIG_X86_INTEL_USERCOPY */
	297
	298	/* Generic arbitrary sized copy. */
3f50dbc1	299	#define __copy_user(to, from, size) \
1da177e4 LT	300	do { \
	301	int __d0, __d1, __d2; \
	302	__asm__ __volatile__( \
	303	" cmp $7,%0\n" \
	304	" jbe 1f\n" \
	305	" movl %1,%0\n" \
	306	" negl %0\n" \
	307	" andl $7,%0\n" \
	308	" subl %0,%3\n" \
	309	"4: rep; movsb\n" \
	310	" movl %3,%0\n" \
	311	" shrl $2,%0\n" \
	312	" andl $3,%3\n" \
	313	" .align 2,0x90\n" \
	314	"0: rep; movsl\n" \
	315	" movl %3,%0\n" \
	316	"1: rep; movsb\n" \
	317	"2:\n" \
	318	".section .fixup,\"ax\"\n" \
	319	"5: addl %3,%0\n" \
	320	" jmp 2b\n" \
	321	"3: lea 0(%3,%0,4),%0\n" \
	322	" jmp 2b\n" \
	323	".previous\n" \
75045f77 JH	324	_ASM_EXTABLE_UA(4b, 5b) \
	325	_ASM_EXTABLE_UA(0b, 3b) \
	326	_ASM_EXTABLE_UA(1b, 2b) \
1da177e4 LT	327	: "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \
	328	: "3"(size), "0"(size), "1"(to), "2"(from) \
	329	: "memory"); \
	330	} while (0)
	331
beba3a20	332	unsigned long __copy_user_ll(void to, const void from, unsigned long n)
1da177e4	333	{
304ec1b0	334	__uaccess_begin_nospec();
1da177e4 LT	335	if (movsl_is_ok(to, from, n))
	336	__copy_user(to, from, n);
	337	else
	338	n = __copy_user_intel(to, from, n);
b5c4ae4f	339	__uaccess_end();
1da177e4 LT	340	return n;
1da177e4 LT	341	}
beba3a20	342	EXPORT_SYMBOL(__copy_user_ll);
7c12d811	343
7c12d811 N	344	unsigned long __copy_from_user_ll_nocache_nozero(void to, const void __user from,
	345	unsigned long n)
	346	{
304ec1b0	347	__uaccess_begin_nospec();
7c12d811	348	#ifdef CONFIG_X86_INTEL_USERCOPY
054efb64	349	if (n > 64 && static_cpu_has(X86_FEATURE_XMM2))
3f50dbc1	350	n = __copy_user_intel_nocache(to, from, n);
7c12d811 N	351	else
	352	__copy_user(to, from, n);
	353	#else
3f50dbc1	354	__copy_user(to, from, n);
7c12d811	355	#endif
b5c4ae4f	356	__uaccess_end();
7c12d811 N	357	return n;
7c12d811 N	358	}
914c8269	359	EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);