[linux-2.6-block.git] / arch / s390 / lib / uaccess.c

// SPDX-License-Identifier: GPL-2.0
/*
 *  Standard user space access functions based on mvcp/mvcs and doing
 *  interesting things in the secondary space mode.
 *
 *    Copyright IBM Corp. 2006,2014
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
 *		 Gerald Schaefer (gerald.schaefer@de.ibm.com)
 */

#include <linux/jump_label.h>
#include <linux/uaccess.h>
#include <linux/export.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <asm/mmu_context.h>
#include <asm/facility.h>

#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES
static DEFINE_STATIC_KEY_FALSE(have_mvcos);

static int __init uaccess_init(void)
{
	if (test_facility(27))
		static_branch_enable(&have_mvcos);
	return 0;
}
early_initcall(uaccess_init);

static inline int copy_with_mvcos(void)
{
	if (static_branch_likely(&have_mvcos))
		return 1;
	return 0;
}
#else
static inline int copy_with_mvcos(void)
{
	return 1;
}
#endif

void set_fs(mm_segment_t fs)
{
	current->thread.mm_segment = fs;
	if (fs == USER_DS) {
		__ctl_load(S390_lowcore.user_asce, 1, 1);
		clear_cpu_flag(CIF_ASCE_PRIMARY);
	} else {
		__ctl_load(S390_lowcore.kernel_asce, 1, 1);
		set_cpu_flag(CIF_ASCE_PRIMARY);
	}
	if (fs & 1) {
		if (fs == USER_DS_SACF)
			__ctl_load(S390_lowcore.user_asce, 7, 7);
		else
			__ctl_load(S390_lowcore.kernel_asce, 7, 7);
		set_cpu_flag(CIF_ASCE_SECONDARY);
	}
}
EXPORT_SYMBOL(set_fs);

mm_segment_t enable_sacf_uaccess(void)
{
	mm_segment_t old_fs;
	unsigned long asce, cr;

	old_fs = current->thread.mm_segment;
	if (old_fs & 1)
		return old_fs;
	current->thread.mm_segment |= 1;
	asce = S390_lowcore.kernel_asce;
	if (likely(old_fs == USER_DS)) {
		__ctl_store(cr, 1, 1);
		if (cr != S390_lowcore.kernel_asce) {
			__ctl_load(S390_lowcore.kernel_asce, 1, 1);
			set_cpu_flag(CIF_ASCE_PRIMARY);
		}
		asce = S390_lowcore.user_asce;
	}
	__ctl_store(cr, 7, 7);
	if (cr != asce) {
		__ctl_load(asce, 7, 7);
		set_cpu_flag(CIF_ASCE_SECONDARY);
	}
	return old_fs;
}
EXPORT_SYMBOL(enable_sacf_uaccess);

void disable_sacf_uaccess(mm_segment_t old_fs)
{
	current->thread.mm_segment = old_fs;
	if (old_fs == USER_DS && test_facility(27)) {
		__ctl_load(S390_lowcore.user_asce, 1, 1);
		clear_cpu_flag(CIF_ASCE_PRIMARY);
	}
}
EXPORT_SYMBOL(disable_sacf_uaccess);

static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr,
						 unsigned long size)
{
	register unsigned long reg0 asm("0") = 0x01UL;
	unsigned long tmp1, tmp2;

	tmp1 = -4096UL;
	asm volatile(
		"0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n"
		"6: jz    4f\n"
		"1: algr  %0,%3\n"
		"   slgr  %1,%3\n"
		"   slgr  %2,%3\n"
		"   j     0b\n"
		"2: la    %4,4095(%1)\n"/* %4 = ptr + 4095 */
		"   nr    %4,%3\n"	/* %4 = (ptr + 4095) & -4096 */
		"   slgr  %4,%1\n"
		"   clgr  %0,%4\n"	/* copy crosses next page boundary? */
		"   jnh   5f\n"
		"3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n"
		"7: slgr  %0,%4\n"
		"   j     5f\n"
		"4: slgr  %0,%0\n"
		"5:\n"
		EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
		: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
		: "d" (reg0) : "cc", "memory");
	return size;
}

static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr,
						unsigned long size)
{
	unsigned long tmp1, tmp2;
	mm_segment_t old_fs;

	old_fs = enable_sacf_uaccess();
	tmp1 = -256UL;
	asm volatile(
		"   sacf  0\n"
		"0: mvcp  0(%0,%2),0(%1),%3\n"
		"7: jz    5f\n"
		"1: algr  %0,%3\n"
		"   la    %1,256(%1)\n"
		"   la    %2,256(%2)\n"
		"2: mvcp  0(%0,%2),0(%1),%3\n"
		"8: jnz   1b\n"
		"   j     5f\n"
		"3: la    %4,255(%1)\n"	/* %4 = ptr + 255 */
		"   lghi  %3,-4096\n"
		"   nr    %4,%3\n"	/* %4 = (ptr + 255) & -4096 */
		"   slgr  %4,%1\n"
		"   clgr  %0,%4\n"	/* copy crosses next page boundary? */
		"   jnh   6f\n"
		"4: mvcp  0(%4,%2),0(%1),%3\n"
		"9: slgr  %0,%4\n"
		"   j     6f\n"
		"5: slgr  %0,%0\n"
		"6: sacf  768\n"
		EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
		EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
		: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
		: : "cc", "memory");
	disable_sacf_uaccess(old_fs);
	return size;
}

unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
	if (copy_with_mvcos())
		return copy_from_user_mvcos(to, from, n);
	return copy_from_user_mvcp(to, from, n);
}
EXPORT_SYMBOL(raw_copy_from_user);

static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
					       unsigned long size)
{
	register unsigned long reg0 asm("0") = 0x010000UL;
	unsigned long tmp1, tmp2;

	tmp1 = -4096UL;
	asm volatile(
		"0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
		"6: jz    4f\n"
		"1: algr  %0,%3\n"
		"   slgr  %1,%3\n"
		"   slgr  %2,%3\n"
		"   j     0b\n"
		"2: la    %4,4095(%1)\n"/* %4 = ptr + 4095 */
		"   nr    %4,%3\n"	/* %4 = (ptr + 4095) & -4096 */
		"   slgr  %4,%1\n"
		"   clgr  %0,%4\n"	/* copy crosses next page boundary? */
		"   jnh   5f\n"
		"3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n"
		"7: slgr  %0,%4\n"
		"   j     5f\n"
		"4: slgr  %0,%0\n"
		"5:\n"
		EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
		: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
		: "d" (reg0) : "cc", "memory");
	return size;
}

static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x,
					      unsigned long size)
{
	unsigned long tmp1, tmp2;
	mm_segment_t old_fs;

	old_fs = enable_sacf_uaccess();
	tmp1 = -256UL;
	asm volatile(
		"   sacf  0\n"
		"0: mvcs  0(%0,%1),0(%2),%3\n"
		"7: jz    5f\n"
		"1: algr  %0,%3\n"
		"   la    %1,256(%1)\n"
		"   la    %2,256(%2)\n"
		"2: mvcs  0(%0,%1),0(%2),%3\n"
		"8: jnz   1b\n"
		"   j     5f\n"
		"3: la    %4,255(%1)\n" /* %4 = ptr + 255 */
		"   lghi  %3,-4096\n"
		"   nr    %4,%3\n"	/* %4 = (ptr + 255) & -4096 */
		"   slgr  %4,%1\n"
		"   clgr  %0,%4\n"	/* copy crosses next page boundary? */
		"   jnh   6f\n"
		"4: mvcs  0(%4,%1),0(%2),%3\n"
		"9: slgr  %0,%4\n"
		"   j     6f\n"
		"5: slgr  %0,%0\n"
		"6: sacf  768\n"
		EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
		EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
		: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
		: : "cc", "memory");
	disable_sacf_uaccess(old_fs);
	return size;
}

unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
	if (copy_with_mvcos())
		return copy_to_user_mvcos(to, from, n);
	return copy_to_user_mvcs(to, from, n);
}
EXPORT_SYMBOL(raw_copy_to_user);

static inline unsigned long copy_in_user_mvcos(void __user *to, const void __user *from,
					       unsigned long size)
{
	register unsigned long reg0 asm("0") = 0x010001UL;
	unsigned long tmp1, tmp2;

	tmp1 = -4096UL;
	/* FIXME: copy with reduced length. */
	asm volatile(
		"0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
		"   jz	  2f\n"
		"1: algr  %0,%3\n"
		"   slgr  %1,%3\n"
		"   slgr  %2,%3\n"
		"   j	  0b\n"
		"2:slgr  %0,%0\n"
		"3: \n"
		EX_TABLE(0b,3b)
		: "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2)
		: "d" (reg0) : "cc", "memory");
	return size;
}

static inline unsigned long copy_in_user_mvc(void __user *to, const void __user *from,
					     unsigned long size)
{
	mm_segment_t old_fs;
	unsigned long tmp1;

	old_fs = enable_sacf_uaccess();
	asm volatile(
		"   sacf  256\n"
		"   aghi  %0,-1\n"
		"   jo	  5f\n"
		"   bras  %3,3f\n"
		"0: aghi  %0,257\n"
		"1: mvc	  0(1,%1),0(%2)\n"
		"   la	  %1,1(%1)\n"
		"   la	  %2,1(%2)\n"
		"   aghi  %0,-1\n"
		"   jnz	  1b\n"
		"   j	  5f\n"
		"2: mvc	  0(256,%1),0(%2)\n"
		"   la	  %1,256(%1)\n"
		"   la	  %2,256(%2)\n"
		"3: aghi  %0,-256\n"
		"   jnm	  2b\n"
		"4: ex	  %0,1b-0b(%3)\n"
		"5: slgr  %0,%0\n"
		"6: sacf  768\n"
		EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
		: "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1)
		: : "cc", "memory");
	disable_sacf_uaccess(old_fs);
	return size;
}

unsigned long raw_copy_in_user(void __user *to, const void __user *from, unsigned long n)
{
	if (copy_with_mvcos())
		return copy_in_user_mvcos(to, from, n);
	return copy_in_user_mvc(to, from, n);
}
EXPORT_SYMBOL(raw_copy_in_user);

static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size)
{
	register unsigned long reg0 asm("0") = 0x010000UL;
	unsigned long tmp1, tmp2;

	tmp1 = -4096UL;
	asm volatile(
		"0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n"
		"   jz	  4f\n"
		"1: algr  %0,%2\n"
		"   slgr  %1,%2\n"
		"   j	  0b\n"
		"2: la	  %3,4095(%1)\n"/* %4 = to + 4095 */
		"   nr	  %3,%2\n"	/* %4 = (to + 4095) & -4096 */
		"   slgr  %3,%1\n"
		"   clgr  %0,%3\n"	/* copy crosses next page boundary? */
		"   jnh	  5f\n"
		"3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n"
		"   slgr  %0,%3\n"
		"   j	  5f\n"
		"4: slgr  %0,%0\n"
		"5:\n"
		EX_TABLE(0b,2b) EX_TABLE(3b,5b)
		: "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2)
		: "a" (empty_zero_page), "d" (reg0) : "cc", "memory");
	return size;
}

static inline unsigned long clear_user_xc(void __user *to, unsigned long size)
{
	mm_segment_t old_fs;
	unsigned long tmp1, tmp2;

	old_fs = enable_sacf_uaccess();
	asm volatile(
		"   sacf  256\n"
		"   aghi  %0,-1\n"
		"   jo    5f\n"
		"   bras  %3,3f\n"
		"   xc    0(1,%1),0(%1)\n"
		"0: aghi  %0,257\n"
		"   la    %2,255(%1)\n" /* %2 = ptr + 255 */
		"   srl   %2,12\n"
		"   sll   %2,12\n"	/* %2 = (ptr + 255) & -4096 */
		"   slgr  %2,%1\n"
		"   clgr  %0,%2\n"	/* clear crosses next page boundary? */
		"   jnh   5f\n"
		"   aghi  %2,-1\n"
		"1: ex    %2,0(%3)\n"
		"   aghi  %2,1\n"
		"   slgr  %0,%2\n"
		"   j     5f\n"
		"2: xc    0(256,%1),0(%1)\n"
		"   la    %1,256(%1)\n"
		"3: aghi  %0,-256\n"
		"   jnm   2b\n"
		"4: ex    %0,0(%3)\n"
		"5: slgr  %0,%0\n"
		"6: sacf  768\n"
		EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
		: "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2)
		: : "cc", "memory");
	disable_sacf_uaccess(old_fs);
	return size;
}

unsigned long __clear_user(void __user *to, unsigned long size)
{
	if (copy_with_mvcos())
			return clear_user_mvcos(to, size);
	return clear_user_xc(to, size);
}
EXPORT_SYMBOL(__clear_user);

static inline unsigned long strnlen_user_srst(const char __user *src,
					      unsigned long size)
{
	register unsigned long reg0 asm("0") = 0;
	unsigned long tmp1, tmp2;

	asm volatile(
		"   la    %2,0(%1)\n"
		"   la    %3,0(%0,%1)\n"
		"   slgr  %0,%0\n"
		"   sacf  256\n"
		"0: srst  %3,%2\n"
		"   jo    0b\n"
		"   la    %0,1(%3)\n"	/* strnlen_user results includes \0 */
		"   slgr  %0,%1\n"
		"1: sacf  768\n"
		EX_TABLE(0b,1b)
		: "+a" (size), "+a" (src), "=a" (tmp1), "=a" (tmp2)
		: "d" (reg0) : "cc", "memory");
	return size;
}

unsigned long __strnlen_user(const char __user *src, unsigned long size)
{
	mm_segment_t old_fs;
	unsigned long len;

	if (unlikely(!size))
		return 0;
	old_fs = enable_sacf_uaccess();
	len = strnlen_user_srst(src, size);
	disable_sacf_uaccess(old_fs);
	return len;
}
EXPORT_SYMBOL(__strnlen_user);

long __strncpy_from_user(char *dst, const char __user *src, long size)
{
	size_t done, len, offset, len_str;

	if (unlikely(size <= 0))
		return 0;
	done = 0;
	do {
		offset = (size_t)src & (L1_CACHE_BYTES - 1);
		len = min(size - done, L1_CACHE_BYTES - offset);
		if (copy_from_user(dst, src, len))
			return -EFAULT;
		len_str = strnlen(dst, len);
		done += len_str;
		src += len_str;
		dst += len_str;
	} while ((len_str == len) && (done < size));
	return done;
}
EXPORT_SYMBOL(__strncpy_from_user);
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
457f2180 HC	2	/*
	3	* Standard user space access functions based on mvcp/mvcs and doing
	4	* interesting things in the secondary space mode.
	5	*
	6	* Copyright IBM Corp. 2006,2014
	7	* Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
	8	* Gerald Schaefer (gerald.schaefer@de.ibm.com)
	9	*/
	10
	11	#include <linux/jump_label.h>
	12	#include <linux/uaccess.h>
	13	#include <linux/export.h>
	14	#include <linux/errno.h>
	15	#include <linux/mm.h>
	16	#include <asm/mmu_context.h>
	17	#include <asm/facility.h>
	18
d66bf801	19	#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES
ed79e946	20	static DEFINE_STATIC_KEY_FALSE(have_mvcos);
457f2180	21
d66bf801 MS	22	static int __init uaccess_init(void)
	23	{
	24	if (test_facility(27))
	25	static_branch_enable(&have_mvcos);
	26	return 0;
	27	}
	28	early_initcall(uaccess_init);
	29
	30	static inline int copy_with_mvcos(void)
	31	{
	32	if (static_branch_likely(&have_mvcos))
	33	return 1;
	34	return 0;
	35	}
	36	#else
	37	static inline int copy_with_mvcos(void)
	38	{
	39	return 1;
	40	}
	41	#endif
	42
0aaba41b MS	43	void set_fs(mm_segment_t fs)
	44	{
	45	current->thread.mm_segment = fs;
	46	if (fs == USER_DS) {
	47	__ctl_load(S390_lowcore.user_asce, 1, 1);
	48	clear_cpu_flag(CIF_ASCE_PRIMARY);
	49	} else {
	50	__ctl_load(S390_lowcore.kernel_asce, 1, 1);
	51	set_cpu_flag(CIF_ASCE_PRIMARY);
	52	}
	53	if (fs & 1) {
	54	if (fs == USER_DS_SACF)
	55	__ctl_load(S390_lowcore.user_asce, 7, 7);
	56	else
	57	__ctl_load(S390_lowcore.kernel_asce, 7, 7);
	58	set_cpu_flag(CIF_ASCE_SECONDARY);
	59	}
	60	}
	61	EXPORT_SYMBOL(set_fs);
	62
	63	mm_segment_t enable_sacf_uaccess(void)
	64	{
	65	mm_segment_t old_fs;
	66	unsigned long asce, cr;
	67
	68	old_fs = current->thread.mm_segment;
	69	if (old_fs & 1)
	70	return old_fs;
	71	current->thread.mm_segment \|= 1;
	72	asce = S390_lowcore.kernel_asce;
	73	if (likely(old_fs == USER_DS)) {
	74	__ctl_store(cr, 1, 1);
	75	if (cr != S390_lowcore.kernel_asce) {
	76	__ctl_load(S390_lowcore.kernel_asce, 1, 1);
	77	set_cpu_flag(CIF_ASCE_PRIMARY);
	78	}
	79	asce = S390_lowcore.user_asce;
	80	}
	81	__ctl_store(cr, 7, 7);
	82	if (cr != asce) {
	83	__ctl_load(asce, 7, 7);
	84	set_cpu_flag(CIF_ASCE_SECONDARY);
	85	}
	86	return old_fs;
	87	}
	88	EXPORT_SYMBOL(enable_sacf_uaccess);
	89
	90	void disable_sacf_uaccess(mm_segment_t old_fs)
	91	{
9f37e797	92	current->thread.mm_segment = old_fs;
0aaba41b MS	93	if (old_fs == USER_DS && test_facility(27)) {
	94	__ctl_load(S390_lowcore.user_asce, 1, 1);
	95	clear_cpu_flag(CIF_ASCE_PRIMARY);
	96	}
0aaba41b MS	97	}
	98	EXPORT_SYMBOL(disable_sacf_uaccess);
	99
457f2180 HC	100	static inline unsigned long copy_from_user_mvcos(void x, const void __user ptr,
	101	unsigned long size)
	102	{
0aaba41b	103	register unsigned long reg0 asm("0") = 0x01UL;
457f2180 HC	104	unsigned long tmp1, tmp2;
	105
	106	tmp1 = -4096UL;
	107	asm volatile(
	108	"0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n"
37096003	109	"6: jz 4f\n"
5a79859a HC	110	"1: algr %0,%3\n"
	111	" slgr %1,%3\n"
	112	" slgr %2,%3\n"
457f2180 HC	113	" j 0b\n"
	114	"2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */
	115	" nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */
5a79859a HC	116	" slgr %4,%1\n"
5a79859a HC	117	" clgr %0,%4\n" /* copy crosses next page boundary? */
37096003	118	" jnh 5f\n"
457f2180	119	"3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n"
37096003 AV	120	"7: slgr %0,%4\n"
	121	" j 5f\n"
	122	"4: slgr %0,%0\n"
	123	"5:\n"
	124	EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
457f2180 HC	125	: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
	126	: "d" (reg0) : "cc", "memory");
	127	return size;
	128	}
	129
	130	static inline unsigned long copy_from_user_mvcp(void x, const void __user ptr,
	131	unsigned long size)
	132	{
	133	unsigned long tmp1, tmp2;
0aaba41b	134	mm_segment_t old_fs;
457f2180	135
0aaba41b	136	old_fs = enable_sacf_uaccess();
457f2180 HC	137	tmp1 = -256UL;
	138	asm volatile(
	139	" sacf 0\n"
	140	"0: mvcp 0(%0,%2),0(%1),%3\n"
37096003	141	"7: jz 5f\n"
5a79859a	142	"1: algr %0,%3\n"
457f2180 HC	143	" la %1,256(%1)\n"
	144	" la %2,256(%2)\n"
	145	"2: mvcp 0(%0,%2),0(%1),%3\n"
37096003 AV	146	"8: jnz 1b\n"
37096003 AV	147	" j 5f\n"
457f2180	148	"3: la %4,255(%1)\n" /* %4 = ptr + 255 */
5a79859a	149	" lghi %3,-4096\n"
457f2180	150	" nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */
5a79859a HC	151	" slgr %4,%1\n"
5a79859a HC	152	" clgr %0,%4\n" /* copy crosses next page boundary? */
37096003	153	" jnh 6f\n"
457f2180	154	"4: mvcp 0(%4,%2),0(%1),%3\n"
37096003 AV	155	"9: slgr %0,%4\n"
	156	" j 6f\n"
	157	"5: slgr %0,%0\n"
	158	"6: sacf 768\n"
	159	EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
	160	EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
457f2180 HC	161	: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
457f2180 HC	162	: : "cc", "memory");
0aaba41b	163	disable_sacf_uaccess(old_fs);
457f2180 HC	164	return size;
	165	}
	166
37096003	167	unsigned long raw_copy_from_user(void to, const void __user from, unsigned long n)
457f2180	168	{
d66bf801	169	if (copy_with_mvcos())
457f2180 HC	170	return copy_from_user_mvcos(to, from, n);
	171	return copy_from_user_mvcp(to, from, n);
	172	}
37096003	173	EXPORT_SYMBOL(raw_copy_from_user);
457f2180 HC	174
	175	static inline unsigned long copy_to_user_mvcos(void __user ptr, const void x,
	176	unsigned long size)
	177	{
0aaba41b	178	register unsigned long reg0 asm("0") = 0x010000UL;
457f2180 HC	179	unsigned long tmp1, tmp2;
	180
	181	tmp1 = -4096UL;
	182	asm volatile(
	183	"0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
	184	"6: jz 4f\n"
5a79859a HC	185	"1: algr %0,%3\n"
	186	" slgr %1,%3\n"
	187	" slgr %2,%3\n"
457f2180 HC	188	" j 0b\n"
	189	"2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */
	190	" nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */
5a79859a HC	191	" slgr %4,%1\n"
5a79859a HC	192	" clgr %0,%4\n" /* copy crosses next page boundary? */
457f2180 HC	193	" jnh 5f\n"
457f2180 HC	194	"3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n"
5a79859a	195	"7: slgr %0,%4\n"
457f2180	196	" j 5f\n"
5a79859a	197	"4: slgr %0,%0\n"
457f2180 HC	198	"5:\n"
	199	EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
	200	: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
	201	: "d" (reg0) : "cc", "memory");
	202	return size;
	203	}
	204
	205	static inline unsigned long copy_to_user_mvcs(void __user ptr, const void x,
	206	unsigned long size)
	207	{
	208	unsigned long tmp1, tmp2;
0aaba41b	209	mm_segment_t old_fs;
457f2180	210
0aaba41b	211	old_fs = enable_sacf_uaccess();
457f2180 HC	212	tmp1 = -256UL;
	213	asm volatile(
	214	" sacf 0\n"
	215	"0: mvcs 0(%0,%1),0(%2),%3\n"
	216	"7: jz 5f\n"
5a79859a	217	"1: algr %0,%3\n"
457f2180 HC	218	" la %1,256(%1)\n"
	219	" la %2,256(%2)\n"
	220	"2: mvcs 0(%0,%1),0(%2),%3\n"
	221	"8: jnz 1b\n"
	222	" j 5f\n"
	223	"3: la %4,255(%1)\n" /* %4 = ptr + 255 */
5a79859a	224	" lghi %3,-4096\n"
457f2180	225	" nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */
5a79859a HC	226	" slgr %4,%1\n"
5a79859a HC	227	" clgr %0,%4\n" /* copy crosses next page boundary? */
457f2180 HC	228	" jnh 6f\n"
457f2180 HC	229	"4: mvcs 0(%4,%1),0(%2),%3\n"
5a79859a	230	"9: slgr %0,%4\n"
457f2180	231	" j 6f\n"
5a79859a	232	"5: slgr %0,%0\n"
457f2180 HC	233	"6: sacf 768\n"
	234	EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
	235	EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
	236	: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
	237	: : "cc", "memory");
0aaba41b	238	disable_sacf_uaccess(old_fs);
457f2180 HC	239	return size;
	240	}
	241
37096003	242	unsigned long raw_copy_to_user(void __user to, const void from, unsigned long n)
457f2180	243	{
d66bf801	244	if (copy_with_mvcos())
457f2180 HC	245	return copy_to_user_mvcos(to, from, n);
	246	return copy_to_user_mvcs(to, from, n);
	247	}
37096003	248	EXPORT_SYMBOL(raw_copy_to_user);
457f2180 HC	249
	250	static inline unsigned long copy_in_user_mvcos(void __user to, const void __user from,
	251	unsigned long size)
	252	{
0aaba41b	253	register unsigned long reg0 asm("0") = 0x010001UL;
457f2180 HC	254	unsigned long tmp1, tmp2;
	255
	256	tmp1 = -4096UL;
	257	/* FIXME: copy with reduced length. */
	258	asm volatile(
	259	"0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
	260	" jz 2f\n"
5a79859a HC	261	"1: algr %0,%3\n"
	262	" slgr %1,%3\n"
	263	" slgr %2,%3\n"
457f2180	264	" j 0b\n"
5a79859a	265	"2:slgr %0,%0\n"
457f2180 HC	266	"3: \n"
	267	EX_TABLE(0b,3b)
	268	: "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2)
	269	: "d" (reg0) : "cc", "memory");
	270	return size;
	271	}
	272
	273	static inline unsigned long copy_in_user_mvc(void __user to, const void __user from,
	274	unsigned long size)
	275	{
0aaba41b	276	mm_segment_t old_fs;
457f2180 HC	277	unsigned long tmp1;
457f2180 HC	278
0aaba41b	279	old_fs = enable_sacf_uaccess();
457f2180 HC	280	asm volatile(
457f2180 HC	281	" sacf 256\n"
5a79859a	282	" aghi %0,-1\n"
457f2180 HC	283	" jo 5f\n"
457f2180 HC	284	" bras %3,3f\n"
5a79859a	285	"0: aghi %0,257\n"
457f2180 HC	286	"1: mvc 0(1,%1),0(%2)\n"
	287	" la %1,1(%1)\n"
	288	" la %2,1(%2)\n"
5a79859a	289	" aghi %0,-1\n"
457f2180 HC	290	" jnz 1b\n"
	291	" j 5f\n"
	292	"2: mvc 0(256,%1),0(%2)\n"
	293	" la %1,256(%1)\n"
	294	" la %2,256(%2)\n"
5a79859a	295	"3: aghi %0,-256\n"
457f2180 HC	296	" jnm 2b\n"
457f2180 HC	297	"4: ex %0,1b-0b(%3)\n"
5a79859a	298	"5: slgr %0,%0\n"
457f2180 HC	299	"6: sacf 768\n"
	300	EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
	301	: "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1)
	302	: : "cc", "memory");
0aaba41b	303	disable_sacf_uaccess(old_fs);
457f2180 HC	304	return size;
	305	}
	306
37096003	307	unsigned long raw_copy_in_user(void __user to, const void __user from, unsigned long n)
457f2180	308	{
d66bf801	309	if (copy_with_mvcos())
457f2180 HC	310	return copy_in_user_mvcos(to, from, n);
	311	return copy_in_user_mvc(to, from, n);
	312	}
37096003	313	EXPORT_SYMBOL(raw_copy_in_user);
457f2180 HC	314
	315	static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size)
	316	{
0aaba41b	317	register unsigned long reg0 asm("0") = 0x010000UL;
457f2180 HC	318	unsigned long tmp1, tmp2;
	319
	320	tmp1 = -4096UL;
	321	asm volatile(
	322	"0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n"
	323	" jz 4f\n"
5a79859a HC	324	"1: algr %0,%2\n"
5a79859a HC	325	" slgr %1,%2\n"
457f2180 HC	326	" j 0b\n"
	327	"2: la %3,4095(%1)\n"/* %4 = to + 4095 */
	328	" nr %3,%2\n" /* %4 = (to + 4095) & -4096 */
5a79859a HC	329	" slgr %3,%1\n"
5a79859a HC	330	" clgr %0,%3\n" /* copy crosses next page boundary? */
457f2180 HC	331	" jnh 5f\n"
457f2180 HC	332	"3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n"
5a79859a	333	" slgr %0,%3\n"
457f2180	334	" j 5f\n"
b8ac5e2f	335	"4: slgr %0,%0\n"
457f2180 HC	336	"5:\n"
	337	EX_TABLE(0b,2b) EX_TABLE(3b,5b)
	338	: "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2)
	339	: "a" (empty_zero_page), "d" (reg0) : "cc", "memory");
	340	return size;
	341	}
	342
	343	static inline unsigned long clear_user_xc(void __user *to, unsigned long size)
	344	{
0aaba41b	345	mm_segment_t old_fs;
457f2180 HC	346	unsigned long tmp1, tmp2;
457f2180 HC	347
0aaba41b	348	old_fs = enable_sacf_uaccess();
457f2180 HC	349	asm volatile(
457f2180 HC	350	" sacf 256\n"
5a79859a	351	" aghi %0,-1\n"
457f2180 HC	352	" jo 5f\n"
	353	" bras %3,3f\n"
	354	" xc 0(1,%1),0(%1)\n"
5a79859a	355	"0: aghi %0,257\n"
457f2180 HC	356	" la %2,255(%1)\n" /* %2 = ptr + 255 */
	357	" srl %2,12\n"
	358	" sll %2,12\n" /* %2 = (ptr + 255) & -4096 */
5a79859a HC	359	" slgr %2,%1\n"
5a79859a HC	360	" clgr %0,%2\n" /* clear crosses next page boundary? */
457f2180	361	" jnh 5f\n"
5a79859a	362	" aghi %2,-1\n"
457f2180	363	"1: ex %2,0(%3)\n"
5a79859a HC	364	" aghi %2,1\n"
5a79859a HC	365	" slgr %0,%2\n"
457f2180 HC	366	" j 5f\n"
	367	"2: xc 0(256,%1),0(%1)\n"
	368	" la %1,256(%1)\n"
5a79859a	369	"3: aghi %0,-256\n"
457f2180 HC	370	" jnm 2b\n"
457f2180 HC	371	"4: ex %0,0(%3)\n"
5a79859a	372	"5: slgr %0,%0\n"
457f2180 HC	373	"6: sacf 768\n"
	374	EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
	375	: "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2)
	376	: : "cc", "memory");
0aaba41b	377	disable_sacf_uaccess(old_fs);
457f2180 HC	378	return size;
	379	}
	380
	381	unsigned long __clear_user(void __user *to, unsigned long size)
	382	{
d66bf801	383	if (copy_with_mvcos())
457f2180 HC	384	return clear_user_mvcos(to, size);
	385	return clear_user_xc(to, size);
	386	}
	387	EXPORT_SYMBOL(__clear_user);
	388
	389	static inline unsigned long strnlen_user_srst(const char __user *src,
	390	unsigned long size)
	391	{
	392	register unsigned long reg0 asm("0") = 0;
	393	unsigned long tmp1, tmp2;
	394
457f2180 HC	395	asm volatile(
	396	" la %2,0(%1)\n"
	397	" la %3,0(%0,%1)\n"
5a79859a	398	" slgr %0,%0\n"
457f2180 HC	399	" sacf 256\n"
	400	"0: srst %3,%2\n"
	401	" jo 0b\n"
	402	" la %0,1(%3)\n" /* strnlen_user results includes \0 */
5a79859a	403	" slgr %0,%1\n"
457f2180 HC	404	"1: sacf 768\n"
	405	EX_TABLE(0b,1b)
	406	: "+a" (size), "+a" (src), "=a" (tmp1), "=a" (tmp2)
	407	: "d" (reg0) : "cc", "memory");
	408	return size;
	409	}
	410
	411	unsigned long __strnlen_user(const char __user *src, unsigned long size)
	412	{
0aaba41b MS	413	mm_segment_t old_fs;
	414	unsigned long len;
	415
fa255f51 HC	416	if (unlikely(!size))
fa255f51 HC	417	return 0;
0aaba41b MS	418	old_fs = enable_sacf_uaccess();
	419	len = strnlen_user_srst(src, size);
	420	disable_sacf_uaccess(old_fs);
	421	return len;
457f2180 HC	422	}
	423	EXPORT_SYMBOL(__strnlen_user);
	424
	425	long __strncpy_from_user(char dst, const char __user src, long size)
	426	{
	427	size_t done, len, offset, len_str;
	428
	429	if (unlikely(size <= 0))
	430	return 0;
	431	done = 0;
	432	do {
f5c8b960 HC	433	offset = (size_t)src & (L1_CACHE_BYTES - 1);
f5c8b960 HC	434	len = min(size - done, L1_CACHE_BYTES - offset);
457f2180 HC	435	if (copy_from_user(dst, src, len))
	436	return -EFAULT;
	437	len_str = strnlen(dst, len);
	438	done += len_str;
	439	src += len_str;
	440	dst += len_str;
	441	} while ((len_str == len) && (done < size));
	442	return done;
	443	}
	444	EXPORT_SYMBOL(__strncpy_from_user);