[linux-2.6-block.git] / arch / arm64 / include / asm / futex.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2012 ARM Ltd.
 */
#ifndef __ASM_FUTEX_H
#define __ASM_FUTEX_H

#include <linux/futex.h>
#include <linux/uaccess.h>

#include <asm/errno.h>

#define FUTEX_MAX_LOOPS	128 /* What's the largest number you can think of? */

#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
do {									\
	unsigned int loops = FUTEX_MAX_LOOPS;				\
									\
	uaccess_enable_privileged();					\
	asm volatile(							\
"	prfm	pstl1strm, %2\n"					\
"1:	ldxr	%w1, %2\n"						\
	insn "\n"							\
"2:	stlxr	%w0, %w3, %2\n"						\
"	cbz	%w0, 3f\n"						\
"	sub	%w4, %w4, %w0\n"					\
"	cbnz	%w4, 1b\n"						\
"	mov	%w0, %w6\n"						\
"3:\n"									\
"	dmb	ish\n"							\
	_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0)				\
	_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w0)				\
	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp),	\
	  "+r" (loops)							\
	: "r" (oparg), "Ir" (-EAGAIN)					\
	: "memory");							\
	uaccess_disable_privileged();					\
} while (0)

static inline int
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr)
{
	int oldval = 0, ret, tmp;
	u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);

	if (!access_ok(_uaddr, sizeof(u32)))
		return -EFAULT;

	switch (op) {
	case FUTEX_OP_SET:
		__futex_atomic_op("mov	%w3, %w5",
				  ret, oldval, uaddr, tmp, oparg);
		break;
	case FUTEX_OP_ADD:
		__futex_atomic_op("add	%w3, %w1, %w5",
				  ret, oldval, uaddr, tmp, oparg);
		break;
	case FUTEX_OP_OR:
		__futex_atomic_op("orr	%w3, %w1, %w5",
				  ret, oldval, uaddr, tmp, oparg);
		break;
	case FUTEX_OP_ANDN:
		__futex_atomic_op("and	%w3, %w1, %w5",
				  ret, oldval, uaddr, tmp, ~oparg);
		break;
	case FUTEX_OP_XOR:
		__futex_atomic_op("eor	%w3, %w1, %w5",
				  ret, oldval, uaddr, tmp, oparg);
		break;
	default:
		ret = -ENOSYS;
	}

	if (!ret)
		*oval = oldval;

	return ret;
}

static inline int
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
			      u32 oldval, u32 newval)
{
	int ret = 0;
	unsigned int loops = FUTEX_MAX_LOOPS;
	u32 val, tmp;
	u32 __user *uaddr;

	if (!access_ok(_uaddr, sizeof(u32)))
		return -EFAULT;

	uaddr = __uaccess_mask_ptr(_uaddr);
	uaccess_enable_privileged();
	asm volatile("// futex_atomic_cmpxchg_inatomic\n"
"	prfm	pstl1strm, %2\n"
"1:	ldxr	%w1, %2\n"
"	sub	%w3, %w1, %w5\n"
"	cbnz	%w3, 4f\n"
"2:	stlxr	%w3, %w6, %2\n"
"	cbz	%w3, 3f\n"
"	sub	%w4, %w4, %w3\n"
"	cbnz	%w4, 1b\n"
"	mov	%w0, %w7\n"
"3:\n"
"	dmb	ish\n"
"4:\n"
	_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0)
	_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0)
	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
	: "r" (oldval), "r" (newval), "Ir" (-EAGAIN)
	: "memory");
	uaccess_disable_privileged();

	if (!ret)
		*uval = val;

	return ret;
}

#endif /* __ASM_FUTEX_H */
Commit	Line	Data
caab277b	1	/* SPDX-License-Identifier: GPL-2.0-only */
6170a974 CM	2	/*
6170a974 CM	3	* Copyright (C) 2012 ARM Ltd.
6170a974 CM	4	*/
	5	#ifndef __ASM_FUTEX_H
	6	#define __ASM_FUTEX_H
	7
6170a974 CM	8	#include <linux/futex.h>
6170a974 CM	9	#include <linux/uaccess.h>
338d4f49	10
6170a974 CM	11	#include <asm/errno.h>
6170a974 CM	12
03110a5c WD	13	#define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */
03110a5c WD	14
6170a974	15	#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \
bd38967d	16	do { \
03110a5c WD	17	unsigned int loops = FUTEX_MAX_LOOPS; \
03110a5c WD	18	\
923e1e7d	19	uaccess_enable_privileged(); \
6170a974	20	asm volatile( \
0ea366f5	21	" prfm pstl1strm, %2\n" \
8e86f0b4	22	"1: ldxr %w1, %2\n" \
6170a974	23	insn "\n" \
045afc24	24	"2: stlxr %w0, %w3, %2\n" \
03110a5c WD	25	" cbz %w0, 3f\n" \
	26	" sub %w4, %w4, %w0\n" \
	27	" cbnz %w4, 1b\n" \
2e77a62c	28	" mov %w0, %w6\n" \
6170a974	29	"3:\n" \
03110a5c	30	" dmb ish\n" \
2e77a62c MR	31	_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0) \
2e77a62c MR	32	_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w0) \
03110a5c WD	33	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp), \
03110a5c WD	34	"+r" (loops) \
2e77a62c	35	: "r" (oparg), "Ir" (-EAGAIN) \
bd38967d	36	: "memory"); \
923e1e7d	37	uaccess_disable_privileged(); \
bd38967d	38	} while (0)
6170a974 CM	39
6170a974 CM	40	static inline int
91b2d344	41	arch_futex_atomic_op_inuser(int op, int oparg, int oval, u32 __user _uaddr)
6170a974	42	{
ff8acf92	43	int oldval = 0, ret, tmp;
91b2d344	44	u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);
6170a974	45
a08971e9 AV	46	if (!access_ok(_uaddr, sizeof(u32)))
a08971e9 AV	47	return -EFAULT;
6170a974 CM	48
	49	switch (op) {
	50	case FUTEX_OP_SET:
03110a5c	51	__futex_atomic_op("mov %w3, %w5",
6170a974 CM	52	ret, oldval, uaddr, tmp, oparg);
	53	break;
	54	case FUTEX_OP_ADD:
03110a5c	55	__futex_atomic_op("add %w3, %w1, %w5",
6170a974 CM	56	ret, oldval, uaddr, tmp, oparg);
	57	break;
	58	case FUTEX_OP_OR:
03110a5c	59	__futex_atomic_op("orr %w3, %w1, %w5",
6170a974 CM	60	ret, oldval, uaddr, tmp, oparg);
	61	break;
	62	case FUTEX_OP_ANDN:
03110a5c	63	__futex_atomic_op("and %w3, %w1, %w5",
6170a974 CM	64	ret, oldval, uaddr, tmp, ~oparg);
	65	break;
	66	case FUTEX_OP_XOR:
03110a5c	67	__futex_atomic_op("eor %w3, %w1, %w5",
6170a974 CM	68	ret, oldval, uaddr, tmp, oparg);
	69	break;
	70	default:
	71	ret = -ENOSYS;
	72	}
	73
30d6e0a4 JS	74	if (!ret)
	75	*oval = oldval;
	76
6170a974 CM	77	return ret;
	78	}
	79
	80	static inline int
91b2d344	81	futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user _uaddr,
6170a974 CM	82	u32 oldval, u32 newval)
	83	{
	84	int ret = 0;
03110a5c	85	unsigned int loops = FUTEX_MAX_LOOPS;
6170a974	86	u32 val, tmp;
91b2d344	87	u32 __user *uaddr;
6170a974	88
96d4f267	89	if (!access_ok(_uaddr, sizeof(u32)))
6170a974 CM	90	return -EFAULT;
6170a974 CM	91
91b2d344	92	uaddr = __uaccess_mask_ptr(_uaddr);
923e1e7d	93	uaccess_enable_privileged();
6170a974	94	asm volatile("// futex_atomic_cmpxchg_inatomic\n"
0ea366f5	95	" prfm pstl1strm, %2\n"
8e86f0b4	96	"1: ldxr %w1, %2\n"
03110a5c WD	97	" sub %w3, %w1, %w5\n"
	98	" cbnz %w3, 4f\n"
	99	"2: stlxr %w3, %w6, %2\n"
	100	" cbz %w3, 3f\n"
	101	" sub %w4, %w4, %w3\n"
	102	" cbnz %w4, 1b\n"
2e77a62c	103	" mov %w0, %w7\n"
6170a974	104	"3:\n"
03110a5c WD	105	" dmb ish\n"
03110a5c WD	106	"4:\n"
2e77a62c MR	107	_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0)
2e77a62c MR	108	_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0)
03110a5c	109	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
2e77a62c	110	: "r" (oldval), "r" (newval), "Ir" (-EAGAIN)
95c41896	111	: "memory");
923e1e7d	112	uaccess_disable_privileged();
6170a974	113
8e4e0ac0 WD	114	if (!ret)
	115	*uval = val;
	116
6170a974 CM	117	return ret;
	118	}
	119
6170a974	120	#endif /* __ASM_FUTEX_H */