[linux-2.6-block.git] / arch / mips / include / asm / futex.h

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (c) 2006  Ralf Baechle (ralf@linux-mips.org)
 */
#ifndef _ASM_FUTEX_H
#define _ASM_FUTEX_H

#ifdef __KERNEL__

#include <linux/futex.h>
#include <linux/uaccess.h>
#include <asm/asm-eva.h>
#include <asm/barrier.h>
#include <asm/compiler.h>
#include <asm/errno.h>
#include <asm/sync.h>

#define arch_futex_atomic_op_inuser arch_futex_atomic_op_inuser
#define futex_atomic_cmpxchg_inatomic futex_atomic_cmpxchg_inatomic
#include <asm-generic/futex.h>

#define __futex_atomic_op(op, insn, ret, oldval, uaddr, oparg)		\
{									\
	if (cpu_has_llsc && IS_ENABLED(CONFIG_WAR_R10000_LLSC)) {	\
		__asm__ __volatile__(					\
		"	.set	push				\n"	\
		"	.set	noat				\n"	\
		"	.set	push				\n"	\
		"	.set	arch=r4000			\n"	\
		"1:	ll	%1, %4	# __futex_atomic_op	\n"	\
		"	.set	pop				\n"	\
		"	" insn	"				\n"	\
		"	.set	arch=r4000			\n"	\
		"2:	sc	$1, %2				\n"	\
		"	beqzl	$1, 1b				\n"	\
		__stringify(__WEAK_LLSC_MB) "			\n"	\
		"3:						\n"	\
		"	.insn					\n"	\
		"	.set	pop				\n"	\
		"	.section .fixup,\"ax\"			\n"	\
		"4:	li	%0, %6				\n"	\
		"	j	3b				\n"	\
		"	.previous				\n"	\
		"	.section __ex_table,\"a\"		\n"	\
		"	"__UA_ADDR "\t1b, 4b			\n"	\
		"	"__UA_ADDR "\t2b, 4b			\n"	\
		"	.previous				\n"	\
		: "=r" (ret), "=&r" (oldval),				\
		  "=" GCC_OFF_SMALL_ASM() (*uaddr)				\
		: "0" (0), GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oparg),	\
		  "i" (-EFAULT)						\
		: "memory");						\
	} else if (cpu_has_llsc) {					\
		__asm__ __volatile__(					\
		"	.set	push				\n"	\
		"	.set	noat				\n"	\
		"	.set	push				\n"	\
		"	.set	"MIPS_ISA_ARCH_LEVEL"		\n"	\
		"	" __SYNC(full, loongson3_war) "		\n"	\
		"1:	"user_ll("%1", "%4")" # __futex_atomic_op\n"	\
		"	.set	pop				\n"	\
		"	" insn	"				\n"	\
		"	.set	"MIPS_ISA_ARCH_LEVEL"		\n"	\
		"2:	"user_sc("$1", "%2")"			\n"	\
		"	beqz	$1, 1b				\n"	\
		__stringify(__WEAK_LLSC_MB) "			\n"	\
		"3:						\n"	\
		"	.insn					\n"	\
		"	.set	pop				\n"	\
		"	.section .fixup,\"ax\"			\n"	\
		"4:	li	%0, %6				\n"	\
		"	j	3b				\n"	\
		"	.previous				\n"	\
		"	.section __ex_table,\"a\"		\n"	\
		"	"__UA_ADDR "\t1b, 4b			\n"	\
		"	"__UA_ADDR "\t2b, 4b			\n"	\
		"	.previous				\n"	\
		: "=r" (ret), "=&r" (oldval),				\
		  "=" GCC_OFF_SMALL_ASM() (*uaddr)				\
		: "0" (0), GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oparg),	\
		  "i" (-EFAULT)						\
		: "memory");						\
	} else {							\
		/* fallback for non-SMP */				\
		ret = futex_atomic_op_inuser_local(op, oparg, oval, uaddr);	\
	}								\
}

static inline int
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
{
	int oldval = 0, ret;

	if (!access_ok(uaddr, sizeof(u32)))
		return -EFAULT;

	switch (op) {
	case FUTEX_OP_SET:
		__futex_atomic_op(op, "move $1, %z5", ret, oldval, uaddr, oparg);
		break;

	case FUTEX_OP_ADD:
		__futex_atomic_op(op, "addu $1, %1, %z5",
				  ret, oldval, uaddr, oparg);
		break;
	case FUTEX_OP_OR:
		__futex_atomic_op(op, "or	$1, %1, %z5",
				  ret, oldval, uaddr, oparg);
		break;
	case FUTEX_OP_ANDN:
		__futex_atomic_op(op, "and	$1, %1, %z5",
				  ret, oldval, uaddr, ~oparg);
		break;
	case FUTEX_OP_XOR:
		__futex_atomic_op(op, "xor	$1, %1, %z5",
				  ret, oldval, uaddr, oparg);
		break;
	default:
		ret = -ENOSYS;
	}

	if (!ret)
		*oval = oldval;

	return ret;
}

static inline int
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
			      u32 oldval, u32 newval)
{
	int ret = 0;
	u32 val;

	if (!access_ok(uaddr, sizeof(u32)))
		return -EFAULT;

	if (cpu_has_llsc && IS_ENABLED(CONFIG_WAR_R10000_LLSC)) {
		__asm__ __volatile__(
		"# futex_atomic_cmpxchg_inatomic			\n"
		"	.set	push					\n"
		"	.set	noat					\n"
		"	.set	push					\n"
		"	.set	arch=r4000				\n"
		"1:	ll	%1, %3					\n"
		"	bne	%1, %z4, 3f				\n"
		"	.set	pop					\n"
		"	move	$1, %z5					\n"
		"	.set	arch=r4000				\n"
		"2:	sc	$1, %2					\n"
		"	beqzl	$1, 1b					\n"
		__stringify(__WEAK_LLSC_MB) "				\n"
		"3:							\n"
		"	.insn						\n"
		"	.set	pop					\n"
		"	.section .fixup,\"ax\"				\n"
		"4:	li	%0, %6					\n"
		"	j	3b					\n"
		"	.previous					\n"
		"	.section __ex_table,\"a\"			\n"
		"	"__UA_ADDR "\t1b, 4b				\n"
		"	"__UA_ADDR "\t2b, 4b				\n"
		"	.previous					\n"
		: "+r" (ret), "=&r" (val), "=" GCC_OFF_SMALL_ASM() (*uaddr)
		: GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oldval), "Jr" (newval),
		  "i" (-EFAULT)
		: "memory");
	} else if (cpu_has_llsc) {
		__asm__ __volatile__(
		"# futex_atomic_cmpxchg_inatomic			\n"
		"	.set	push					\n"
		"	.set	noat					\n"
		"	.set	push					\n"
		"	.set	"MIPS_ISA_ARCH_LEVEL"			\n"
		"	" __SYNC(full, loongson3_war) "			\n"
		"1:	"user_ll("%1", "%3")"				\n"
		"	bne	%1, %z4, 3f				\n"
		"	.set	pop					\n"
		"	move	$1, %z5					\n"
		"	.set	"MIPS_ISA_ARCH_LEVEL"			\n"
		"2:	"user_sc("$1", "%2")"				\n"
		"	beqz	$1, 1b					\n"
		"3:	" __SYNC_ELSE(full, loongson3_war, __WEAK_LLSC_MB) "\n"
		"	.insn						\n"
		"	.set	pop					\n"
		"	.section .fixup,\"ax\"				\n"
		"4:	li	%0, %6					\n"
		"	j	3b					\n"
		"	.previous					\n"
		"	.section __ex_table,\"a\"			\n"
		"	"__UA_ADDR "\t1b, 4b				\n"
		"	"__UA_ADDR "\t2b, 4b				\n"
		"	.previous					\n"
		: "+r" (ret), "=&r" (val), "=" GCC_OFF_SMALL_ASM() (*uaddr)
		: GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oldval), "Jr" (newval),
		  "i" (-EFAULT)
		: "memory");
	} else {
		return futex_atomic_cmpxchg_inatomic_local(uval, uaddr, oldval, newval);
	}

	*uval = val;
	return ret;
}

#endif
#endif /* _ASM_FUTEX_H */
Commit	Line	Data
	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
	6	* Copyright (c) 2006 Ralf Baechle (ralf@linux-mips.org)
	7	*/
	8	#ifndef _ASM_FUTEX_H
	9	#define _ASM_FUTEX_H
	10
	11	#ifdef __KERNEL__
	12
	13	#include <linux/futex.h>
	14	#include <linux/uaccess.h>
	15	#include <asm/asm-eva.h>
	16	#include <asm/barrier.h>
	17	#include <asm/compiler.h>
	18	#include <asm/errno.h>
	19	#include <asm/sync.h>
	20
	21	#define arch_futex_atomic_op_inuser arch_futex_atomic_op_inuser
	22	#define futex_atomic_cmpxchg_inatomic futex_atomic_cmpxchg_inatomic
	23	#include <asm-generic/futex.h>
	24
	25	#define __futex_atomic_op(op, insn, ret, oldval, uaddr, oparg) \
	26	{ \
	27	if (cpu_has_llsc && IS_ENABLED(CONFIG_WAR_R10000_LLSC)) { \
	28	__asm__ __volatile__( \
	29	" .set push \n" \
	30	" .set noat \n" \
	31	" .set push \n" \
	32	" .set arch=r4000 \n" \
	33	"1: ll %1, %4 # __futex_atomic_op \n" \
	34	" .set pop \n" \
	35	" " insn " \n" \
	36	" .set arch=r4000 \n" \
	37	"2: sc $1, %2 \n" \
	38	" beqzl $1, 1b \n" \
	39	__stringify(__WEAK_LLSC_MB) " \n" \
	40	"3: \n" \
	41	" .insn \n" \
	42	" .set pop \n" \
	43	" .section .fixup,\"ax\" \n" \
	44	"4: li %0, %6 \n" \
	45	" j 3b \n" \
	46	" .previous \n" \
	47	" .section __ex_table,\"a\" \n" \
	48	" "__UA_ADDR "\t1b, 4b \n" \
	49	" "__UA_ADDR "\t2b, 4b \n" \
	50	" .previous \n" \
	51	: "=r" (ret), "=&r" (oldval), \
	52	"=" GCC_OFF_SMALL_ASM() (*uaddr) \
	53	: "0" (0), GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oparg), \
	54	"i" (-EFAULT) \
	55	: "memory"); \
	56	} else if (cpu_has_llsc) { \
	57	__asm__ __volatile__( \
	58	" .set push \n" \
	59	" .set noat \n" \
	60	" .set push \n" \
	61	" .set "MIPS_ISA_ARCH_LEVEL" \n" \
	62	" " __SYNC(full, loongson3_war) " \n" \
	63	"1: "user_ll("%1", "%4")" # __futex_atomic_op\n" \
	64	" .set pop \n" \
	65	" " insn " \n" \
	66	" .set "MIPS_ISA_ARCH_LEVEL" \n" \
	67	"2: "user_sc("$1", "%2")" \n" \
	68	" beqz $1, 1b \n" \
	69	__stringify(__WEAK_LLSC_MB) " \n" \
	70	"3: \n" \
	71	" .insn \n" \
	72	" .set pop \n" \
	73	" .section .fixup,\"ax\" \n" \
	74	"4: li %0, %6 \n" \
	75	" j 3b \n" \
	76	" .previous \n" \
	77	" .section __ex_table,\"a\" \n" \
	78	" "__UA_ADDR "\t1b, 4b \n" \
	79	" "__UA_ADDR "\t2b, 4b \n" \
	80	" .previous \n" \
	81	: "=r" (ret), "=&r" (oldval), \
	82	"=" GCC_OFF_SMALL_ASM() (*uaddr) \
	83	: "0" (0), GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oparg), \
	84	"i" (-EFAULT) \
	85	: "memory"); \
	86	} else { \
	87	/* fallback for non-SMP */ \
	88	ret = futex_atomic_op_inuser_local(op, oparg, oval, uaddr); \
	89	} \
	90	}
	91
	92	static inline int
	93	arch_futex_atomic_op_inuser(int op, int oparg, int oval, u32 __user uaddr)
	94	{
	95	int oldval = 0, ret;
	96
	97	if (!access_ok(uaddr, sizeof(u32)))
	98	return -EFAULT;
	99
	100	switch (op) {
	101	case FUTEX_OP_SET:
	102	__futex_atomic_op(op, "move $1, %z5", ret, oldval, uaddr, oparg);
	103	break;
	104
	105	case FUTEX_OP_ADD:
	106	__futex_atomic_op(op, "addu $1, %1, %z5",
	107	ret, oldval, uaddr, oparg);
	108	break;
	109	case FUTEX_OP_OR:
	110	__futex_atomic_op(op, "or $1, %1, %z5",
	111	ret, oldval, uaddr, oparg);
	112	break;
	113	case FUTEX_OP_ANDN:
	114	__futex_atomic_op(op, "and $1, %1, %z5",
	115	ret, oldval, uaddr, ~oparg);
	116	break;
	117	case FUTEX_OP_XOR:
	118	__futex_atomic_op(op, "xor $1, %1, %z5",
	119	ret, oldval, uaddr, oparg);
	120	break;
	121	default:
	122	ret = -ENOSYS;
	123	}
	124
	125	if (!ret)
	126	*oval = oldval;
	127
	128	return ret;
	129	}
	130
	131	static inline int
	132	futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user uaddr,
	133	u32 oldval, u32 newval)
	134	{
	135	int ret = 0;
	136	u32 val;
	137
	138	if (!access_ok(uaddr, sizeof(u32)))
	139	return -EFAULT;
	140
	141	if (cpu_has_llsc && IS_ENABLED(CONFIG_WAR_R10000_LLSC)) {
	142	__asm__ __volatile__(
	143	"# futex_atomic_cmpxchg_inatomic \n"
	144	" .set push \n"
	145	" .set noat \n"
	146	" .set push \n"
	147	" .set arch=r4000 \n"
	148	"1: ll %1, %3 \n"
	149	" bne %1, %z4, 3f \n"
	150	" .set pop \n"
	151	" move $1, %z5 \n"
	152	" .set arch=r4000 \n"
	153	"2: sc $1, %2 \n"
	154	" beqzl $1, 1b \n"
	155	__stringify(__WEAK_LLSC_MB) " \n"
	156	"3: \n"
	157	" .insn \n"
	158	" .set pop \n"
	159	" .section .fixup,\"ax\" \n"
	160	"4: li %0, %6 \n"
	161	" j 3b \n"
	162	" .previous \n"
	163	" .section __ex_table,\"a\" \n"
	164	" "__UA_ADDR "\t1b, 4b \n"
	165	" "__UA_ADDR "\t2b, 4b \n"
	166	" .previous \n"
	167	: "+r" (ret), "=&r" (val), "=" GCC_OFF_SMALL_ASM() (*uaddr)
	168	: GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oldval), "Jr" (newval),
	169	"i" (-EFAULT)
	170	: "memory");
	171	} else if (cpu_has_llsc) {
	172	__asm__ __volatile__(
	173	"# futex_atomic_cmpxchg_inatomic \n"
	174	" .set push \n"
	175	" .set noat \n"
	176	" .set push \n"
	177	" .set "MIPS_ISA_ARCH_LEVEL" \n"
	178	" " __SYNC(full, loongson3_war) " \n"
	179	"1: "user_ll("%1", "%3")" \n"
	180	" bne %1, %z4, 3f \n"
	181	" .set pop \n"
	182	" move $1, %z5 \n"
	183	" .set "MIPS_ISA_ARCH_LEVEL" \n"
	184	"2: "user_sc("$1", "%2")" \n"
	185	" beqz $1, 1b \n"
	186	"3: " __SYNC_ELSE(full, loongson3_war, __WEAK_LLSC_MB) "\n"
	187	" .insn \n"
	188	" .set pop \n"
	189	" .section .fixup,\"ax\" \n"
	190	"4: li %0, %6 \n"
	191	" j 3b \n"
	192	" .previous \n"
	193	" .section __ex_table,\"a\" \n"
	194	" "__UA_ADDR "\t1b, 4b \n"
	195	" "__UA_ADDR "\t2b, 4b \n"
	196	" .previous \n"
	197	: "+r" (ret), "=&r" (val), "=" GCC_OFF_SMALL_ASM() (*uaddr)
	198	: GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oldval), "Jr" (newval),
	199	"i" (-EFAULT)
	200	: "memory");
	201	} else {
	202	return futex_atomic_cmpxchg_inatomic_local(uval, uaddr, oldval, newval);
	203	}
	204
	205	*uval = val;
	206	return ret;
	207	}
	208
	209	#endif
	210	#endif /* _ASM_FUTEX_H */