[linux-block.git] / arch / powerpc / include / asm / cmpxchg.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_CMPXCHG_H_
#define _ASM_POWERPC_CMPXCHG_H_

#ifdef __KERNEL__
#include <linux/compiler.h>
#include <asm/synch.h>
#include <asm/asm-compat.h>
#include <linux/bug.h>

#ifdef __BIG_ENDIAN
#define BITOFF_CAL(size, off)	((sizeof(u32) - size - off) * BITS_PER_BYTE)
#else
#define BITOFF_CAL(size, off)	(off * BITS_PER_BYTE)
#endif

#define XCHG_GEN(type, sfx, cl)				\
static inline u32 __xchg_##type##sfx(volatile void *p, u32 val)	\
{								\
	unsigned int prev, prev_mask, tmp, bitoff, off;		\
								\
	off = (unsigned long)p % sizeof(u32);			\
	bitoff = BITOFF_CAL(sizeof(type), off);			\
	p -= off;						\
	val <<= bitoff;						\
	prev_mask = (u32)(type)-1 << bitoff;			\
								\
	__asm__ __volatile__(					\
"1:	lwarx   %0,0,%3\n"					\
"	andc	%1,%0,%5\n"					\
"	or	%1,%1,%4\n"					\
	PPC405_ERR77(0,%3)					\
"	stwcx.	%1,0,%3\n"					\
"	bne-	1b\n"						\
	: "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p)		\
	: "r" (p), "r" (val), "r" (prev_mask)			\
	: "cc", cl);						\
								\
	return prev >> bitoff;					\
}

#define CMPXCHG_GEN(type, sfx, br, br2, cl)			\
static inline							\
u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new)	\
{								\
	unsigned int prev, prev_mask, tmp, bitoff, off;		\
								\
	off = (unsigned long)p % sizeof(u32);			\
	bitoff = BITOFF_CAL(sizeof(type), off);			\
	p -= off;						\
	old <<= bitoff;						\
	new <<= bitoff;						\
	prev_mask = (u32)(type)-1 << bitoff;			\
								\
	__asm__ __volatile__(					\
	br							\
"1:	lwarx   %0,0,%3\n"					\
"	and	%1,%0,%6\n"					\
"	cmpw	0,%1,%4\n"					\
"	bne-	2f\n"						\
"	andc	%1,%0,%6\n"					\
"	or	%1,%1,%5\n"					\
	PPC405_ERR77(0,%3)					\
"	stwcx.  %1,0,%3\n"					\
"	bne-    1b\n"						\
	br2							\
	"\n"							\
"2:"								\
	: "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p)		\
	: "r" (p), "r" (old), "r" (new), "r" (prev_mask)	\
	: "cc", cl);						\
								\
	return prev >> bitoff;					\
}

/*
 * Atomic exchange
 *
 * Changes the memory location '*p' to be val and returns
 * the previous value stored there.
 */

XCHG_GEN(u8, _local, "memory");
XCHG_GEN(u8, _relaxed, "cc");
XCHG_GEN(u16, _local, "memory");
XCHG_GEN(u16, _relaxed, "cc");

static __always_inline unsigned long
__xchg_u32_local(volatile void *p, unsigned long val)
{
	unsigned long prev;

	__asm__ __volatile__(
"1:	lwarx	%0,0,%2 \n"
	PPC405_ERR77(0,%2)
"	stwcx.	%3,0,%2 \n\
	bne-	1b"
	: "=&r" (prev), "+m" (*(volatile unsigned int *)p)
	: "r" (p), "r" (val)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__xchg_u32_relaxed(u32 *p, unsigned long val)
{
	unsigned long prev;

	__asm__ __volatile__(
"1:	lwarx	%0,0,%2\n"
	PPC405_ERR77(0, %2)
"	stwcx.	%3,0,%2\n"
"	bne-	1b"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (val)
	: "cc");

	return prev;
}

#ifdef CONFIG_PPC64
static __always_inline unsigned long
__xchg_u64_local(volatile void *p, unsigned long val)
{
	unsigned long prev;

	__asm__ __volatile__(
"1:	ldarx	%0,0,%2 \n"
	PPC405_ERR77(0,%2)
"	stdcx.	%3,0,%2 \n\
	bne-	1b"
	: "=&r" (prev), "+m" (*(volatile unsigned long *)p)
	: "r" (p), "r" (val)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__xchg_u64_relaxed(u64 *p, unsigned long val)
{
	unsigned long prev;

	__asm__ __volatile__(
"1:	ldarx	%0,0,%2\n"
	PPC405_ERR77(0, %2)
"	stdcx.	%3,0,%2\n"
"	bne-	1b"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (val)
	: "cc");

	return prev;
}
#endif

static __always_inline unsigned long
__xchg_local(void *ptr, unsigned long x, unsigned int size)
{
	switch (size) {
	case 1:
		return __xchg_u8_local(ptr, x);
	case 2:
		return __xchg_u16_local(ptr, x);
	case 4:
		return __xchg_u32_local(ptr, x);
#ifdef CONFIG_PPC64
	case 8:
		return __xchg_u64_local(ptr, x);
#endif
	}
	BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg");
	return x;
}

static __always_inline unsigned long
__xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
{
	switch (size) {
	case 1:
		return __xchg_u8_relaxed(ptr, x);
	case 2:
		return __xchg_u16_relaxed(ptr, x);
	case 4:
		return __xchg_u32_relaxed(ptr, x);
#ifdef CONFIG_PPC64
	case 8:
		return __xchg_u64_relaxed(ptr, x);
#endif
	}
	BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
	return x;
}
#define xchg_local(ptr,x)						     \
  ({									     \
     __typeof__(*(ptr)) _x_ = (x);					     \
     (__typeof__(*(ptr))) __xchg_local((ptr),				     \
     		(unsigned long)_x_, sizeof(*(ptr))); 			     \
  })

#define xchg_relaxed(ptr, x)						\
({									\
	__typeof__(*(ptr)) _x_ = (x);					\
	(__typeof__(*(ptr))) __xchg_relaxed((ptr),			\
			(unsigned long)_x_, sizeof(*(ptr)));		\
})
/*
 * Compare and exchange - if *p == old, set it to new,
 * and return the old value of *p.
 */

CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
CMPXCHG_GEN(u8, _local, , , "memory");
CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
CMPXCHG_GEN(u8, _relaxed, , , "cc");
CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
CMPXCHG_GEN(u16, _local, , , "memory");
CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
CMPXCHG_GEN(u16, _relaxed, , , "cc");

static __always_inline unsigned long
__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
{
	unsigned int prev;

	__asm__ __volatile__ (
	PPC_ATOMIC_ENTRY_BARRIER
"1:	lwarx	%0,0,%2		# __cmpxchg_u32\n\
	cmpw	0,%0,%3\n\
	bne-	2f\n"
	PPC405_ERR77(0,%2)
"	stwcx.	%4,0,%2\n\
	bne-	1b"
	PPC_ATOMIC_EXIT_BARRIER
	"\n\
2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
			unsigned long new)
{
	unsigned int prev;

	__asm__ __volatile__ (
"1:	lwarx	%0,0,%2		# __cmpxchg_u32\n\
	cmpw	0,%0,%3\n\
	bne-	2f\n"
	PPC405_ERR77(0,%2)
"	stwcx.	%4,0,%2\n\
	bne-	1b"
	"\n\
2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	lwarx	%0,0,%2		# __cmpxchg_u32_relaxed\n"
"	cmpw	0,%0,%3\n"
"	bne-	2f\n"
	PPC405_ERR77(0, %2)
"	stwcx.	%4,0,%2\n"
"	bne-	1b\n"
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc");

	return prev;
}

/*
 * cmpxchg family don't have order guarantee if cmp part fails, therefore we
 * can avoid superfluous barriers if we use assembly code to implement
 * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
 * cmpxchg_release() because that will result in putting a barrier in the
 * middle of a ll/sc loop, which is probably a bad idea. For example, this
 * might cause the conditional store more likely to fail.
 */
static __always_inline unsigned long
__cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	lwarx	%0,0,%2		# __cmpxchg_u32_acquire\n"
"	cmpw	0,%0,%3\n"
"	bne-	2f\n"
	PPC405_ERR77(0, %2)
"	stwcx.	%4,0,%2\n"
"	bne-	1b\n"
	PPC_ACQUIRE_BARRIER
	"\n"
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

#ifdef CONFIG_PPC64
static __always_inline unsigned long
__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
	PPC_ATOMIC_ENTRY_BARRIER
"1:	ldarx	%0,0,%2		# __cmpxchg_u64\n\
	cmpd	0,%0,%3\n\
	bne-	2f\n\
	stdcx.	%4,0,%2\n\
	bne-	1b"
	PPC_ATOMIC_EXIT_BARRIER
	"\n\
2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
			unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	ldarx	%0,0,%2		# __cmpxchg_u64\n\
	cmpd	0,%0,%3\n\
	bne-	2f\n\
	stdcx.	%4,0,%2\n\
	bne-	1b"
	"\n\
2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	ldarx	%0,0,%2		# __cmpxchg_u64_relaxed\n"
"	cmpd	0,%0,%3\n"
"	bne-	2f\n"
"	stdcx.	%4,0,%2\n"
"	bne-	1b\n"
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	ldarx	%0,0,%2		# __cmpxchg_u64_acquire\n"
"	cmpd	0,%0,%3\n"
"	bne-	2f\n"
"	stdcx.	%4,0,%2\n"
"	bne-	1b\n"
	PPC_ACQUIRE_BARRIER
	"\n"
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}
#endif

static __always_inline unsigned long
__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
	  unsigned int size)
{
	switch (size) {
	case 1:
		return __cmpxchg_u8(ptr, old, new);
	case 2:
		return __cmpxchg_u16(ptr, old, new);
	case 4:
		return __cmpxchg_u32(ptr, old, new);
#ifdef CONFIG_PPC64
	case 8:
		return __cmpxchg_u64(ptr, old, new);
#endif
	}
	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
	return old;
}

static __always_inline unsigned long
__cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
	  unsigned int size)
{
	switch (size) {
	case 1:
		return __cmpxchg_u8_local(ptr, old, new);
	case 2:
		return __cmpxchg_u16_local(ptr, old, new);
	case 4:
		return __cmpxchg_u32_local(ptr, old, new);
#ifdef CONFIG_PPC64
	case 8:
		return __cmpxchg_u64_local(ptr, old, new);
#endif
	}
	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
	return old;
}

static __always_inline unsigned long
__cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
		  unsigned int size)
{
	switch (size) {
	case 1:
		return __cmpxchg_u8_relaxed(ptr, old, new);
	case 2:
		return __cmpxchg_u16_relaxed(ptr, old, new);
	case 4:
		return __cmpxchg_u32_relaxed(ptr, old, new);
#ifdef CONFIG_PPC64
	case 8:
		return __cmpxchg_u64_relaxed(ptr, old, new);
#endif
	}
	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
	return old;
}

static __always_inline unsigned long
__cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
		  unsigned int size)
{
	switch (size) {
	case 1:
		return __cmpxchg_u8_acquire(ptr, old, new);
	case 2:
		return __cmpxchg_u16_acquire(ptr, old, new);
	case 4:
		return __cmpxchg_u32_acquire(ptr, old, new);
#ifdef CONFIG_PPC64
	case 8:
		return __cmpxchg_u64_acquire(ptr, old, new);
#endif
	}
	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
	return old;
}
#define cmpxchg(ptr, o, n)						 \
  ({									 \
     __typeof__(*(ptr)) _o_ = (o);					 \
     __typeof__(*(ptr)) _n_ = (n);					 \
     (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,		 \
				    (unsigned long)_n_, sizeof(*(ptr))); \
  })


#define cmpxchg_local(ptr, o, n)					 \
  ({									 \
     __typeof__(*(ptr)) _o_ = (o);					 \
     __typeof__(*(ptr)) _n_ = (n);					 \
     (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_,	 \
				    (unsigned long)_n_, sizeof(*(ptr))); \
  })

#define cmpxchg_relaxed(ptr, o, n)					\
({									\
	__typeof__(*(ptr)) _o_ = (o);					\
	__typeof__(*(ptr)) _n_ = (n);					\
	(__typeof__(*(ptr))) __cmpxchg_relaxed((ptr),			\
			(unsigned long)_o_, (unsigned long)_n_,		\
			sizeof(*(ptr)));				\
})

#define cmpxchg_acquire(ptr, o, n)					\
({									\
	__typeof__(*(ptr)) _o_ = (o);					\
	__typeof__(*(ptr)) _n_ = (n);					\
	(__typeof__(*(ptr))) __cmpxchg_acquire((ptr),			\
			(unsigned long)_o_, (unsigned long)_n_,		\
			sizeof(*(ptr)));				\
})
#ifdef CONFIG_PPC64
#define cmpxchg64(ptr, o, n)						\
  ({									\
	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
	cmpxchg((ptr), (o), (n));					\
  })
#define cmpxchg64_local(ptr, o, n)					\
  ({									\
	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
	cmpxchg_local((ptr), (o), (n));					\
  })
#define cmpxchg64_relaxed(ptr, o, n)					\
({									\
	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
	cmpxchg_relaxed((ptr), (o), (n));				\
})
#define cmpxchg64_acquire(ptr, o, n)					\
({									\
	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
	cmpxchg_acquire((ptr), (o), (n));				\
})
#else
#include <asm-generic/cmpxchg-local.h>
#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
#endif

#endif /* __KERNEL__ */
#endif /* _ASM_POWERPC_CMPXCHG_H_ */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
ae3a197e DH	2	#ifndef _ASM_POWERPC_CMPXCHG_H_
	3	#define _ASM_POWERPC_CMPXCHG_H_
	4
	5	#ifdef __KERNEL__
	6	#include <linux/compiler.h>
	7	#include <asm/synch.h>
	8	#include <asm/asm-compat.h>
10d8b148	9	#include <linux/bug.h>
ae3a197e	10
d0563a12 PX	11	#ifdef __BIG_ENDIAN
	12	#define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE)
	13	#else
	14	#define BITOFF_CAL(size, off) (off * BITS_PER_BYTE)
	15	#endif
	16
	17	#define XCHG_GEN(type, sfx, cl) \
da58b23c	18	static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \
d0563a12 PX	19	{ \
	20	unsigned int prev, prev_mask, tmp, bitoff, off; \
	21	\
	22	off = (unsigned long)p % sizeof(u32); \
	23	bitoff = BITOFF_CAL(sizeof(type), off); \
	24	p -= off; \
	25	val <<= bitoff; \
	26	prev_mask = (u32)(type)-1 << bitoff; \
	27	\
	28	__asm__ __volatile__( \
	29	"1: lwarx %0,0,%3\n" \
	30	" andc %1,%0,%5\n" \
	31	" or %1,%1,%4\n" \
	32	PPC405_ERR77(0,%3) \
	33	" stwcx. %1,0,%3\n" \
	34	" bne- 1b\n" \
	35	: "=&r" (prev), "=&r" (tmp), "+m" ((u32)p) \
	36	: "r" (p), "r" (val), "r" (prev_mask) \
	37	: "cc", cl); \
	38	\
	39	return prev >> bitoff; \
	40	}
	41
	42	#define CMPXCHG_GEN(type, sfx, br, br2, cl) \
	43	static inline \
da58b23c	44	u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \
d0563a12 PX	45	{ \
	46	unsigned int prev, prev_mask, tmp, bitoff, off; \
	47	\
	48	off = (unsigned long)p % sizeof(u32); \
	49	bitoff = BITOFF_CAL(sizeof(type), off); \
	50	p -= off; \
	51	old <<= bitoff; \
	52	new <<= bitoff; \
	53	prev_mask = (u32)(type)-1 << bitoff; \
	54	\
	55	__asm__ __volatile__( \
	56	br \
	57	"1: lwarx %0,0,%3\n" \
	58	" and %1,%0,%6\n" \
	59	" cmpw 0,%1,%4\n" \
	60	" bne- 2f\n" \
	61	" andc %1,%0,%6\n" \
	62	" or %1,%1,%5\n" \
	63	PPC405_ERR77(0,%3) \
	64	" stwcx. %1,0,%3\n" \
	65	" bne- 1b\n" \
	66	br2 \
	67	"\n" \
	68	"2:" \
	69	: "=&r" (prev), "=&r" (tmp), "+m" ((u32)p) \
	70	: "r" (p), "r" (old), "r" (new), "r" (prev_mask) \
	71	: "cc", cl); \
	72	\
	73	return prev >> bitoff; \
	74	}
	75
ae3a197e DH	76	/*
	77	* Atomic exchange
	78	*
26760fc1	79	* Changes the memory location '*p' to be val and returns
ae3a197e DH	80	* the previous value stored there.
ae3a197e DH	81	*/
26760fc1	82
d0563a12 PX	83	XCHG_GEN(u8, _local, "memory");
	84	XCHG_GEN(u8, _relaxed, "cc");
	85	XCHG_GEN(u16, _local, "memory");
	86	XCHG_GEN(u16, _relaxed, "cc");
	87
ae3a197e	88	static __always_inline unsigned long
26760fc1	89	__xchg_u32_local(volatile void *p, unsigned long val)
ae3a197e DH	90	{
	91	unsigned long prev;
	92
	93	__asm__ __volatile__(
ae3a197e DH	94	"1: lwarx %0,0,%2 \n"
	95	PPC405_ERR77(0,%2)
	96	" stwcx. %3,0,%2 \n\
	97	bne- 1b"
ae3a197e DH	98	: "=&r" (prev), "+m" ((volatile unsigned int )p)
	99	: "r" (p), "r" (val)
	100	: "cc", "memory");
	101
	102	return prev;
	103	}
	104
ae3a197e	105	static __always_inline unsigned long
26760fc1	106	__xchg_u32_relaxed(u32 *p, unsigned long val)
ae3a197e DH	107	{
	108	unsigned long prev;
	109
	110	__asm__ __volatile__(
26760fc1 BF	111	"1: lwarx %0,0,%2\n"
	112	PPC405_ERR77(0, %2)
	113	" stwcx. %3,0,%2\n"
	114	" bne- 1b"
	115	: "=&r" (prev), "+m" (*p)
ae3a197e	116	: "r" (p), "r" (val)
26760fc1	117	: "cc");
ae3a197e DH	118
	119	return prev;
	120	}
	121
	122	#ifdef CONFIG_PPC64
	123	static __always_inline unsigned long
26760fc1	124	__xchg_u64_local(volatile void *p, unsigned long val)
ae3a197e DH	125	{
	126	unsigned long prev;
	127
	128	__asm__ __volatile__(
ae3a197e DH	129	"1: ldarx %0,0,%2 \n"
	130	PPC405_ERR77(0,%2)
	131	" stdcx. %3,0,%2 \n\
	132	bne- 1b"
ae3a197e DH	133	: "=&r" (prev), "+m" ((volatile unsigned long )p)
	134	: "r" (p), "r" (val)
	135	: "cc", "memory");
	136
	137	return prev;
	138	}
	139
	140	static __always_inline unsigned long
26760fc1	141	__xchg_u64_relaxed(u64 *p, unsigned long val)
ae3a197e DH	142	{
	143	unsigned long prev;
	144
	145	__asm__ __volatile__(
26760fc1 BF	146	"1: ldarx %0,0,%2\n"
	147	PPC405_ERR77(0, %2)
	148	" stdcx. %3,0,%2\n"
	149	" bne- 1b"
	150	: "=&r" (prev), "+m" (*p)
ae3a197e	151	: "r" (p), "r" (val)
26760fc1	152	: "cc");
ae3a197e DH	153
	154	return prev;
	155	}
	156	#endif
	157
ae3a197e	158	static __always_inline unsigned long
d0563a12	159	__xchg_local(void *ptr, unsigned long x, unsigned int size)
ae3a197e DH	160	{
ae3a197e DH	161	switch (size) {
d0563a12 PX	162	case 1:
	163	return __xchg_u8_local(ptr, x);
	164	case 2:
	165	return __xchg_u16_local(ptr, x);
ae3a197e	166	case 4:
26760fc1	167	return __xchg_u32_local(ptr, x);
ae3a197e DH	168	#ifdef CONFIG_PPC64
ae3a197e DH	169	case 8:
26760fc1	170	return __xchg_u64_local(ptr, x);
ae3a197e DH	171	#endif
ae3a197e DH	172	}
10d8b148	173	BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg");
ae3a197e DH	174	return x;
	175	}
	176
	177	static __always_inline unsigned long
26760fc1	178	__xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
ae3a197e DH	179	{
ae3a197e DH	180	switch (size) {
d0563a12 PX	181	case 1:
	182	return __xchg_u8_relaxed(ptr, x);
	183	case 2:
	184	return __xchg_u16_relaxed(ptr, x);
ae3a197e	185	case 4:
26760fc1	186	return __xchg_u32_relaxed(ptr, x);
ae3a197e DH	187	#ifdef CONFIG_PPC64
ae3a197e DH	188	case 8:
26760fc1	189	return __xchg_u64_relaxed(ptr, x);
ae3a197e DH	190	#endif
ae3a197e DH	191	}
10d8b148	192	BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
ae3a197e DH	193	return x;
ae3a197e DH	194	}
ae3a197e DH	195	#define xchg_local(ptr,x) \
	196	({ \
	197	__typeof__(*(ptr)) _x_ = (x); \
	198	(__typeof__(*(ptr))) __xchg_local((ptr), \
	199	(unsigned long)_x_, sizeof(*(ptr))); \
	200	})
	201
26760fc1 BF	202	#define xchg_relaxed(ptr, x) \
	203	({ \
	204	__typeof__(*(ptr)) _x_ = (x); \
	205	(__typeof__(*(ptr))) __xchg_relaxed((ptr), \
	206	(unsigned long)_x_, sizeof(*(ptr))); \
	207	})
ae3a197e DH	208	/*
	209	* Compare and exchange - if *p == old, set it to new,
	210	* and return the old value of *p.
	211	*/
ae3a197e	212
d0563a12 PX	213	CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
	214	CMPXCHG_GEN(u8, _local, , , "memory");
	215	CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
	216	CMPXCHG_GEN(u8, _relaxed, , , "cc");
	217	CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
	218	CMPXCHG_GEN(u16, _local, , , "memory");
	219	CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
	220	CMPXCHG_GEN(u16, _relaxed, , , "cc");
	221
ae3a197e DH	222	static __always_inline unsigned long
	223	__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
	224	{
	225	unsigned int prev;
	226
	227	__asm__ __volatile__ (
81d7a329	228	PPC_ATOMIC_ENTRY_BARRIER
ae3a197e DH	229	"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
	230	cmpw 0,%0,%3\n\
	231	bne- 2f\n"
	232	PPC405_ERR77(0,%2)
	233	" stwcx. %4,0,%2\n\
	234	bne- 1b"
81d7a329	235	PPC_ATOMIC_EXIT_BARRIER
ae3a197e DH	236	"\n\
	237	2:"
	238	: "=&r" (prev), "+m" (*p)
	239	: "r" (p), "r" (old), "r" (new)
	240	: "cc", "memory");
	241
	242	return prev;
	243	}
	244
	245	static __always_inline unsigned long
	246	__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
	247	unsigned long new)
	248	{
	249	unsigned int prev;
	250
	251	__asm__ __volatile__ (
	252	"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
	253	cmpw 0,%0,%3\n\
	254	bne- 2f\n"
	255	PPC405_ERR77(0,%2)
	256	" stwcx. %4,0,%2\n\
	257	bne- 1b"
	258	"\n\
	259	2:"
	260	: "=&r" (prev), "+m" (*p)
	261	: "r" (p), "r" (old), "r" (new)
	262	: "cc", "memory");
	263
	264	return prev;
	265	}
	266
56c08e6d BF	267	static __always_inline unsigned long
	268	__cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
	269	{
	270	unsigned long prev;
	271
	272	__asm__ __volatile__ (
	273	"1: lwarx %0,0,%2 # __cmpxchg_u32_relaxed\n"
	274	" cmpw 0,%0,%3\n"
	275	" bne- 2f\n"
	276	PPC405_ERR77(0, %2)
	277	" stwcx. %4,0,%2\n"
	278	" bne- 1b\n"
	279	"2:"
	280	: "=&r" (prev), "+m" (*p)
	281	: "r" (p), "r" (old), "r" (new)
	282	: "cc");
	283
	284	return prev;
	285	}
	286
	287	/*
	288	* cmpxchg family don't have order guarantee if cmp part fails, therefore we
	289	* can avoid superfluous barriers if we use assembly code to implement
	290	* cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
	291	* cmpxchg_release() because that will result in putting a barrier in the
	292	* middle of a ll/sc loop, which is probably a bad idea. For example, this
	293	* might cause the conditional store more likely to fail.
	294	*/
	295	static __always_inline unsigned long
	296	__cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
	297	{
	298	unsigned long prev;
	299
	300	__asm__ __volatile__ (
	301	"1: lwarx %0,0,%2 # __cmpxchg_u32_acquire\n"
	302	" cmpw 0,%0,%3\n"
	303	" bne- 2f\n"
	304	PPC405_ERR77(0, %2)
	305	" stwcx. %4,0,%2\n"
	306	" bne- 1b\n"
	307	PPC_ACQUIRE_BARRIER
	308	"\n"
	309	"2:"
	310	: "=&r" (prev), "+m" (*p)
	311	: "r" (p), "r" (old), "r" (new)
	312	: "cc", "memory");
	313
	314	return prev;
	315	}
	316
ae3a197e DH	317	#ifdef CONFIG_PPC64
	318	static __always_inline unsigned long
	319	__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
	320	{
	321	unsigned long prev;
	322
	323	__asm__ __volatile__ (
81d7a329	324	PPC_ATOMIC_ENTRY_BARRIER
ae3a197e DH	325	"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
	326	cmpd 0,%0,%3\n\
	327	bne- 2f\n\
	328	stdcx. %4,0,%2\n\
	329	bne- 1b"
81d7a329	330	PPC_ATOMIC_EXIT_BARRIER
ae3a197e DH	331	"\n\
	332	2:"
	333	: "=&r" (prev), "+m" (*p)
	334	: "r" (p), "r" (old), "r" (new)
	335	: "cc", "memory");
	336
	337	return prev;
	338	}
	339
	340	static __always_inline unsigned long
	341	__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
	342	unsigned long new)
	343	{
	344	unsigned long prev;
	345
	346	__asm__ __volatile__ (
	347	"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
	348	cmpd 0,%0,%3\n\
	349	bne- 2f\n\
	350	stdcx. %4,0,%2\n\
	351	bne- 1b"
	352	"\n\
	353	2:"
	354	: "=&r" (prev), "+m" (*p)
	355	: "r" (p), "r" (old), "r" (new)
	356	: "cc", "memory");
	357
	358	return prev;
	359	}
56c08e6d BF	360
	361	static __always_inline unsigned long
	362	__cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
	363	{
	364	unsigned long prev;
	365
	366	__asm__ __volatile__ (
	367	"1: ldarx %0,0,%2 # __cmpxchg_u64_relaxed\n"
	368	" cmpd 0,%0,%3\n"
	369	" bne- 2f\n"
	370	" stdcx. %4,0,%2\n"
	371	" bne- 1b\n"
	372	"2:"
	373	: "=&r" (prev), "+m" (*p)
	374	: "r" (p), "r" (old), "r" (new)
	375	: "cc");
	376
	377	return prev;
	378	}
	379
	380	static __always_inline unsigned long
	381	__cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
	382	{
	383	unsigned long prev;
	384
	385	__asm__ __volatile__ (
	386	"1: ldarx %0,0,%2 # __cmpxchg_u64_acquire\n"
	387	" cmpd 0,%0,%3\n"
	388	" bne- 2f\n"
	389	" stdcx. %4,0,%2\n"
	390	" bne- 1b\n"
	391	PPC_ACQUIRE_BARRIER
	392	"\n"
	393	"2:"
	394	: "=&r" (prev), "+m" (*p)
	395	: "r" (p), "r" (old), "r" (new)
	396	: "cc", "memory");
	397
	398	return prev;
	399	}
ae3a197e DH	400	#endif
ae3a197e DH	401
ae3a197e	402	static __always_inline unsigned long
da58b23c	403	__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
ae3a197e DH	404	unsigned int size)
	405	{
	406	switch (size) {
d0563a12 PX	407	case 1:
	408	return __cmpxchg_u8(ptr, old, new);
	409	case 2:
	410	return __cmpxchg_u16(ptr, old, new);
ae3a197e DH	411	case 4:
	412	return __cmpxchg_u32(ptr, old, new);
	413	#ifdef CONFIG_PPC64
	414	case 8:
	415	return __cmpxchg_u64(ptr, old, new);
	416	#endif
	417	}
10d8b148	418	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
ae3a197e DH	419	return old;
	420	}
	421
	422	static __always_inline unsigned long
d0563a12	423	__cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
ae3a197e DH	424	unsigned int size)
	425	{
	426	switch (size) {
d0563a12 PX	427	case 1:
	428	return __cmpxchg_u8_local(ptr, old, new);
	429	case 2:
	430	return __cmpxchg_u16_local(ptr, old, new);
ae3a197e DH	431	case 4:
	432	return __cmpxchg_u32_local(ptr, old, new);
	433	#ifdef CONFIG_PPC64
	434	case 8:
	435	return __cmpxchg_u64_local(ptr, old, new);
	436	#endif
	437	}
10d8b148	438	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
ae3a197e DH	439	return old;
	440	}
	441
56c08e6d BF	442	static __always_inline unsigned long
	443	__cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
	444	unsigned int size)
	445	{
	446	switch (size) {
d0563a12 PX	447	case 1:
	448	return __cmpxchg_u8_relaxed(ptr, old, new);
	449	case 2:
	450	return __cmpxchg_u16_relaxed(ptr, old, new);
56c08e6d BF	451	case 4:
	452	return __cmpxchg_u32_relaxed(ptr, old, new);
	453	#ifdef CONFIG_PPC64
	454	case 8:
	455	return __cmpxchg_u64_relaxed(ptr, old, new);
	456	#endif
	457	}
10d8b148	458	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
56c08e6d BF	459	return old;
	460	}
	461
	462	static __always_inline unsigned long
	463	__cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
	464	unsigned int size)
	465	{
	466	switch (size) {
d0563a12 PX	467	case 1:
	468	return __cmpxchg_u8_acquire(ptr, old, new);
	469	case 2:
	470	return __cmpxchg_u16_acquire(ptr, old, new);
56c08e6d BF	471	case 4:
	472	return __cmpxchg_u32_acquire(ptr, old, new);
	473	#ifdef CONFIG_PPC64
	474	case 8:
	475	return __cmpxchg_u64_acquire(ptr, old, new);
	476	#endif
	477	}
10d8b148	478	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
56c08e6d BF	479	return old;
56c08e6d BF	480	}
ae3a197e DH	481	#define cmpxchg(ptr, o, n) \
	482	({ \
	483	__typeof__(*(ptr)) _o_ = (o); \
	484	__typeof__(*(ptr)) _n_ = (n); \
	485	(__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
	486	(unsigned long)_n_, sizeof(*(ptr))); \
	487	})
	488
	489
	490	#define cmpxchg_local(ptr, o, n) \
	491	({ \
	492	__typeof__(*(ptr)) _o_ = (o); \
	493	__typeof__(*(ptr)) _n_ = (n); \
	494	(__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \
	495	(unsigned long)_n_, sizeof(*(ptr))); \
	496	})
	497
56c08e6d BF	498	#define cmpxchg_relaxed(ptr, o, n) \
	499	({ \
	500	__typeof__(*(ptr)) _o_ = (o); \
	501	__typeof__(*(ptr)) _n_ = (n); \
	502	(__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
	503	(unsigned long)_o_, (unsigned long)_n_, \
	504	sizeof(*(ptr))); \
	505	})
	506
	507	#define cmpxchg_acquire(ptr, o, n) \
	508	({ \
	509	__typeof__(*(ptr)) _o_ = (o); \
	510	__typeof__(*(ptr)) _n_ = (n); \
	511	(__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
	512	(unsigned long)_o_, (unsigned long)_n_, \
	513	sizeof(*(ptr))); \
	514	})
ae3a197e DH	515	#ifdef CONFIG_PPC64
	516	#define cmpxchg64(ptr, o, n) \
	517	({ \
	518	BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
	519	cmpxchg((ptr), (o), (n)); \
	520	})
	521	#define cmpxchg64_local(ptr, o, n) \
	522	({ \
	523	BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
	524	cmpxchg_local((ptr), (o), (n)); \
	525	})
56c08e6d BF	526	#define cmpxchg64_relaxed(ptr, o, n) \
	527	({ \
	528	BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
	529	cmpxchg_relaxed((ptr), (o), (n)); \
	530	})
	531	#define cmpxchg64_acquire(ptr, o, n) \
	532	({ \
	533	BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
	534	cmpxchg_acquire((ptr), (o), (n)); \
	535	})
ae3a197e DH	536	#else
	537	#include <asm-generic/cmpxchg-local.h>
	538	#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
	539	#endif
	540
	541	#endif /* __KERNEL__ */
	542	#endif /* _ASM_POWERPC_CMPXCHG_H_ */