[linux-2.6-block.git] / arch / powerpc / include / asm / cmpxchg.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_CMPXCHG_H_
#define _ASM_POWERPC_CMPXCHG_H_

#ifdef __KERNEL__
#include <linux/compiler.h>
#include <asm/synch.h>
#include <linux/bug.h>

#ifdef __BIG_ENDIAN
#define BITOFF_CAL(size, off)	((sizeof(u32) - size - off) * BITS_PER_BYTE)
#else
#define BITOFF_CAL(size, off)	(off * BITS_PER_BYTE)
#endif

#define XCHG_GEN(type, sfx, cl)				\
static inline u32 __xchg_##type##sfx(volatile void *p, u32 val)	\
{								\
	unsigned int prev, prev_mask, tmp, bitoff, off;		\
								\
	off = (unsigned long)p % sizeof(u32);			\
	bitoff = BITOFF_CAL(sizeof(type), off);			\
	p -= off;						\
	val <<= bitoff;						\
	prev_mask = (u32)(type)-1 << bitoff;			\
								\
	__asm__ __volatile__(					\
"1:	lwarx   %0,0,%3\n"					\
"	andc	%1,%0,%5\n"					\
"	or	%1,%1,%4\n"					\
"	stwcx.	%1,0,%3\n"					\
"	bne-	1b\n"						\
	: "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p)		\
	: "r" (p), "r" (val), "r" (prev_mask)			\
	: "cc", cl);						\
								\
	return prev >> bitoff;					\
}

#define CMPXCHG_GEN(type, sfx, br, br2, cl)			\
static inline							\
u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new)	\
{								\
	unsigned int prev, prev_mask, tmp, bitoff, off;		\
								\
	off = (unsigned long)p % sizeof(u32);			\
	bitoff = BITOFF_CAL(sizeof(type), off);			\
	p -= off;						\
	old <<= bitoff;						\
	new <<= bitoff;						\
	prev_mask = (u32)(type)-1 << bitoff;			\
								\
	__asm__ __volatile__(					\
	br							\
"1:	lwarx   %0,0,%3\n"					\
"	and	%1,%0,%6\n"					\
"	cmpw	0,%1,%4\n"					\
"	bne-	2f\n"						\
"	andc	%1,%0,%6\n"					\
"	or	%1,%1,%5\n"					\
"	stwcx.  %1,0,%3\n"					\
"	bne-    1b\n"						\
	br2							\
	"\n"							\
"2:"								\
	: "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p)		\
	: "r" (p), "r" (old), "r" (new), "r" (prev_mask)	\
	: "cc", cl);						\
								\
	return prev >> bitoff;					\
}

/*
 * Atomic exchange
 *
 * Changes the memory location '*p' to be val and returns
 * the previous value stored there.
 */

#ifndef CONFIG_PPC_HAS_LBARX_LHARX
XCHG_GEN(u8, _local, "memory");
XCHG_GEN(u8, _relaxed, "cc");
XCHG_GEN(u16, _local, "memory");
XCHG_GEN(u16, _relaxed, "cc");
#else
static __always_inline unsigned long
__xchg_u8_local(volatile void *p, unsigned long val)
{
	unsigned long prev;

	__asm__ __volatile__(
"1:	lbarx	%0,0,%2		# __xchg_u8_local\n"
"	stbcx.	%3,0,%2 \n"
"	bne-	1b"
	: "=&r" (prev), "+m" (*(volatile unsigned char *)p)
	: "r" (p), "r" (val)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__xchg_u8_relaxed(u8 *p, unsigned long val)
{
	unsigned long prev;

	__asm__ __volatile__(
"1:	lbarx	%0,0,%2		# __xchg_u8_relaxed\n"
"	stbcx.	%3,0,%2\n"
"	bne-	1b"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (val)
	: "cc");

	return prev;
}

static __always_inline unsigned long
__xchg_u16_local(volatile void *p, unsigned long val)
{
	unsigned long prev;

	__asm__ __volatile__(
"1:	lharx	%0,0,%2		# __xchg_u16_local\n"
"	sthcx.	%3,0,%2\n"
"	bne-	1b"
	: "=&r" (prev), "+m" (*(volatile unsigned short *)p)
	: "r" (p), "r" (val)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__xchg_u16_relaxed(u16 *p, unsigned long val)
{
	unsigned long prev;

	__asm__ __volatile__(
"1:	lharx	%0,0,%2		# __xchg_u16_relaxed\n"
"	sthcx.	%3,0,%2\n"
"	bne-	1b"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (val)
	: "cc");

	return prev;
}
#endif

static __always_inline unsigned long
__xchg_u32_local(volatile void *p, unsigned long val)
{
	unsigned long prev;

	__asm__ __volatile__(
"1:	lwarx	%0,0,%2 \n"
"	stwcx.	%3,0,%2 \n\
	bne-	1b"
	: "=&r" (prev), "+m" (*(volatile unsigned int *)p)
	: "r" (p), "r" (val)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__xchg_u32_relaxed(u32 *p, unsigned long val)
{
	unsigned long prev;

	__asm__ __volatile__(
"1:	lwarx	%0,0,%2\n"
"	stwcx.	%3,0,%2\n"
"	bne-	1b"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (val)
	: "cc");

	return prev;
}

#ifdef CONFIG_PPC64
static __always_inline unsigned long
__xchg_u64_local(volatile void *p, unsigned long val)
{
	unsigned long prev;

	__asm__ __volatile__(
"1:	ldarx	%0,0,%2 \n"
"	stdcx.	%3,0,%2 \n\
	bne-	1b"
	: "=&r" (prev), "+m" (*(volatile unsigned long *)p)
	: "r" (p), "r" (val)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__xchg_u64_relaxed(u64 *p, unsigned long val)
{
	unsigned long prev;

	__asm__ __volatile__(
"1:	ldarx	%0,0,%2\n"
"	stdcx.	%3,0,%2\n"
"	bne-	1b"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (val)
	: "cc");

	return prev;
}
#endif

static __always_inline unsigned long
__xchg_local(void *ptr, unsigned long x, unsigned int size)
{
	switch (size) {
	case 1:
		return __xchg_u8_local(ptr, x);
	case 2:
		return __xchg_u16_local(ptr, x);
	case 4:
		return __xchg_u32_local(ptr, x);
#ifdef CONFIG_PPC64
	case 8:
		return __xchg_u64_local(ptr, x);
#endif
	}
	BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
	return x;
}

static __always_inline unsigned long
__xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
{
	switch (size) {
	case 1:
		return __xchg_u8_relaxed(ptr, x);
	case 2:
		return __xchg_u16_relaxed(ptr, x);
	case 4:
		return __xchg_u32_relaxed(ptr, x);
#ifdef CONFIG_PPC64
	case 8:
		return __xchg_u64_relaxed(ptr, x);
#endif
	}
	BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_relaxed");
	return x;
}
#define arch_xchg_local(ptr,x)						     \
  ({									     \
     __typeof__(*(ptr)) _x_ = (x);					     \
     (__typeof__(*(ptr))) __xchg_local((ptr),				     \
     		(unsigned long)_x_, sizeof(*(ptr))); 			     \
  })

#define arch_xchg_relaxed(ptr, x)					\
({									\
	__typeof__(*(ptr)) _x_ = (x);					\
	(__typeof__(*(ptr))) __xchg_relaxed((ptr),			\
			(unsigned long)_x_, sizeof(*(ptr)));		\
})

/*
 * Compare and exchange - if *p == old, set it to new,
 * and return the old value of *p.
 */
#ifndef CONFIG_PPC_HAS_LBARX_LHARX
CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
CMPXCHG_GEN(u8, _local, , , "memory");
CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
CMPXCHG_GEN(u8, _relaxed, , , "cc");
CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
CMPXCHG_GEN(u16, _local, , , "memory");
CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
CMPXCHG_GEN(u16, _relaxed, , , "cc");
#else
static __always_inline unsigned long
__cmpxchg_u8(volatile unsigned char *p, unsigned long old, unsigned long new)
{
	unsigned int prev;

	__asm__ __volatile__ (
	PPC_ATOMIC_ENTRY_BARRIER
"1:	lbarx	%0,0,%2		# __cmpxchg_u8\n"
"	cmpw	0,%0,%3\n"
"	bne-	2f\n"
"	stbcx.	%4,0,%2\n"
"	bne-	1b"
	PPC_ATOMIC_EXIT_BARRIER
	"\n\
2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u8_local(volatile unsigned char *p, unsigned long old,
			unsigned long new)
{
	unsigned int prev;

	__asm__ __volatile__ (
"1:	lbarx	%0,0,%2		# __cmpxchg_u8_local\n"
"	cmpw	0,%0,%3\n"
"	bne-	2f\n"
"	stbcx.	%4,0,%2\n"
"	bne-	1b\n"
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u8_relaxed(u8 *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	lbarx	%0,0,%2		# __cmpxchg_u8_relaxed\n"
"	cmpw	0,%0,%3\n"
"	bne-	2f\n"
"	stbcx.	%4,0,%2\n"
"	bne-	1b\n"
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u8_acquire(u8 *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	lbarx	%0,0,%2		# __cmpxchg_u8_acquire\n"
"	cmpw	0,%0,%3\n"
"	bne-	2f\n"
"	stbcx.	%4,0,%2\n"
"	bne-	1b\n"
	PPC_ACQUIRE_BARRIER
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u16(volatile unsigned short *p, unsigned long old, unsigned long new)
{
	unsigned int prev;

	__asm__ __volatile__ (
	PPC_ATOMIC_ENTRY_BARRIER
"1:	lharx	%0,0,%2		# __cmpxchg_u16\n"
"	cmpw	0,%0,%3\n"
"	bne-	2f\n"
"	sthcx.	%4,0,%2\n"
"	bne-	1b\n"
	PPC_ATOMIC_EXIT_BARRIER
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u16_local(volatile unsigned short *p, unsigned long old,
			unsigned long new)
{
	unsigned int prev;

	__asm__ __volatile__ (
"1:	lharx	%0,0,%2		# __cmpxchg_u16_local\n"
"	cmpw	0,%0,%3\n"
"	bne-	2f\n"
"	sthcx.	%4,0,%2\n"
"	bne-	1b"
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u16_relaxed(u16 *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	lharx	%0,0,%2		# __cmpxchg_u16_relaxed\n"
"	cmpw	0,%0,%3\n"
"	bne-	2f\n"
"	sthcx.	%4,0,%2\n"
"	bne-	1b\n"
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u16_acquire(u16 *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	lharx	%0,0,%2		# __cmpxchg_u16_acquire\n"
"	cmpw	0,%0,%3\n"
"	bne-	2f\n"
"	sthcx.	%4,0,%2\n"
"	bne-	1b\n"
	PPC_ACQUIRE_BARRIER
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}
#endif

static __always_inline unsigned long
__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
{
	unsigned int prev;

	__asm__ __volatile__ (
	PPC_ATOMIC_ENTRY_BARRIER
"1:	lwarx	%0,0,%2		# __cmpxchg_u32\n\
	cmpw	0,%0,%3\n\
	bne-	2f\n"
"	stwcx.	%4,0,%2\n\
	bne-	1b"
	PPC_ATOMIC_EXIT_BARRIER
	"\n\
2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
			unsigned long new)
{
	unsigned int prev;

	__asm__ __volatile__ (
"1:	lwarx	%0,0,%2		# __cmpxchg_u32\n\
	cmpw	0,%0,%3\n\
	bne-	2f\n"
"	stwcx.	%4,0,%2\n\
	bne-	1b"
	"\n\
2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	lwarx	%0,0,%2		# __cmpxchg_u32_relaxed\n"
"	cmpw	0,%0,%3\n"
"	bne-	2f\n"
"	stwcx.	%4,0,%2\n"
"	bne-	1b\n"
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc");

	return prev;
}

/*
 * cmpxchg family don't have order guarantee if cmp part fails, therefore we
 * can avoid superfluous barriers if we use assembly code to implement
 * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
 * cmpxchg_release() because that will result in putting a barrier in the
 * middle of a ll/sc loop, which is probably a bad idea. For example, this
 * might cause the conditional store more likely to fail.
 */
static __always_inline unsigned long
__cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	lwarx	%0,0,%2		# __cmpxchg_u32_acquire\n"
"	cmpw	0,%0,%3\n"
"	bne-	2f\n"
"	stwcx.	%4,0,%2\n"
"	bne-	1b\n"
	PPC_ACQUIRE_BARRIER
	"\n"
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

#ifdef CONFIG_PPC64
static __always_inline unsigned long
__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
	PPC_ATOMIC_ENTRY_BARRIER
"1:	ldarx	%0,0,%2		# __cmpxchg_u64\n\
	cmpd	0,%0,%3\n\
	bne-	2f\n\
	stdcx.	%4,0,%2\n\
	bne-	1b"
	PPC_ATOMIC_EXIT_BARRIER
	"\n\
2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
			unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	ldarx	%0,0,%2		# __cmpxchg_u64\n\
	cmpd	0,%0,%3\n\
	bne-	2f\n\
	stdcx.	%4,0,%2\n\
	bne-	1b"
	"\n\
2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	ldarx	%0,0,%2		# __cmpxchg_u64_relaxed\n"
"	cmpd	0,%0,%3\n"
"	bne-	2f\n"
"	stdcx.	%4,0,%2\n"
"	bne-	1b\n"
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc");

	return prev;
}

static __always_inline unsigned long
__cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
{
	unsigned long prev;

	__asm__ __volatile__ (
"1:	ldarx	%0,0,%2		# __cmpxchg_u64_acquire\n"
"	cmpd	0,%0,%3\n"
"	bne-	2f\n"
"	stdcx.	%4,0,%2\n"
"	bne-	1b\n"
	PPC_ACQUIRE_BARRIER
	"\n"
"2:"
	: "=&r" (prev), "+m" (*p)
	: "r" (p), "r" (old), "r" (new)
	: "cc", "memory");

	return prev;
}
#endif

static __always_inline unsigned long
__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
	  unsigned int size)
{
	switch (size) {
	case 1:
		return __cmpxchg_u8(ptr, old, new);
	case 2:
		return __cmpxchg_u16(ptr, old, new);
	case 4:
		return __cmpxchg_u32(ptr, old, new);
#ifdef CONFIG_PPC64
	case 8:
		return __cmpxchg_u64(ptr, old, new);
#endif
	}
	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
	return old;
}

static __always_inline unsigned long
__cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
	  unsigned int size)
{
	switch (size) {
	case 1:
		return __cmpxchg_u8_local(ptr, old, new);
	case 2:
		return __cmpxchg_u16_local(ptr, old, new);
	case 4:
		return __cmpxchg_u32_local(ptr, old, new);
#ifdef CONFIG_PPC64
	case 8:
		return __cmpxchg_u64_local(ptr, old, new);
#endif
	}
	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
	return old;
}

static __always_inline unsigned long
__cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
		  unsigned int size)
{
	switch (size) {
	case 1:
		return __cmpxchg_u8_relaxed(ptr, old, new);
	case 2:
		return __cmpxchg_u16_relaxed(ptr, old, new);
	case 4:
		return __cmpxchg_u32_relaxed(ptr, old, new);
#ifdef CONFIG_PPC64
	case 8:
		return __cmpxchg_u64_relaxed(ptr, old, new);
#endif
	}
	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
	return old;
}

static __always_inline unsigned long
__cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
		  unsigned int size)
{
	switch (size) {
	case 1:
		return __cmpxchg_u8_acquire(ptr, old, new);
	case 2:
		return __cmpxchg_u16_acquire(ptr, old, new);
	case 4:
		return __cmpxchg_u32_acquire(ptr, old, new);
#ifdef CONFIG_PPC64
	case 8:
		return __cmpxchg_u64_acquire(ptr, old, new);
#endif
	}
	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
	return old;
}
#define arch_cmpxchg(ptr, o, n)						 \
  ({									 \
     __typeof__(*(ptr)) _o_ = (o);					 \
     __typeof__(*(ptr)) _n_ = (n);					 \
     (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,		 \
				    (unsigned long)_n_, sizeof(*(ptr))); \
  })


#define arch_cmpxchg_local(ptr, o, n)					 \
  ({									 \
     __typeof__(*(ptr)) _o_ = (o);					 \
     __typeof__(*(ptr)) _n_ = (n);					 \
     (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_,	 \
				    (unsigned long)_n_, sizeof(*(ptr))); \
  })

#define arch_cmpxchg_relaxed(ptr, o, n)					\
({									\
	__typeof__(*(ptr)) _o_ = (o);					\
	__typeof__(*(ptr)) _n_ = (n);					\
	(__typeof__(*(ptr))) __cmpxchg_relaxed((ptr),			\
			(unsigned long)_o_, (unsigned long)_n_,		\
			sizeof(*(ptr)));				\
})

#define arch_cmpxchg_acquire(ptr, o, n)					\
({									\
	__typeof__(*(ptr)) _o_ = (o);					\
	__typeof__(*(ptr)) _n_ = (n);					\
	(__typeof__(*(ptr))) __cmpxchg_acquire((ptr),			\
			(unsigned long)_o_, (unsigned long)_n_,		\
			sizeof(*(ptr)));				\
})
#ifdef CONFIG_PPC64
#define arch_cmpxchg64(ptr, o, n)					\
  ({									\
	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
	arch_cmpxchg((ptr), (o), (n));					\
  })
#define arch_cmpxchg64_local(ptr, o, n)					\
  ({									\
	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
	arch_cmpxchg_local((ptr), (o), (n));				\
  })
#define arch_cmpxchg64_relaxed(ptr, o, n)				\
({									\
	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
	arch_cmpxchg_relaxed((ptr), (o), (n));				\
})
#define arch_cmpxchg64_acquire(ptr, o, n)				\
({									\
	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
	arch_cmpxchg_acquire((ptr), (o), (n));				\
})
#else
#include <asm-generic/cmpxchg-local.h>
#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
#endif

#endif /* __KERNEL__ */
#endif /* _ASM_POWERPC_CMPXCHG_H_ */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
ae3a197e DH	2	#ifndef _ASM_POWERPC_CMPXCHG_H_
	3	#define _ASM_POWERPC_CMPXCHG_H_
	4
	5	#ifdef __KERNEL__
	6	#include <linux/compiler.h>
	7	#include <asm/synch.h>
10d8b148	8	#include <linux/bug.h>
ae3a197e	9
d0563a12 PX	10	#ifdef __BIG_ENDIAN
	11	#define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE)
	12	#else
	13	#define BITOFF_CAL(size, off) (off * BITS_PER_BYTE)
	14	#endif
	15
	16	#define XCHG_GEN(type, sfx, cl) \
da58b23c	17	static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \
d0563a12 PX	18	{ \
	19	unsigned int prev, prev_mask, tmp, bitoff, off; \
	20	\
	21	off = (unsigned long)p % sizeof(u32); \
	22	bitoff = BITOFF_CAL(sizeof(type), off); \
	23	p -= off; \
	24	val <<= bitoff; \
	25	prev_mask = (u32)(type)-1 << bitoff; \
	26	\
	27	__asm__ __volatile__( \
	28	"1: lwarx %0,0,%3\n" \
	29	" andc %1,%0,%5\n" \
	30	" or %1,%1,%4\n" \
d0563a12 PX	31	" stwcx. %1,0,%3\n" \
	32	" bne- 1b\n" \
	33	: "=&r" (prev), "=&r" (tmp), "+m" ((u32)p) \
	34	: "r" (p), "r" (val), "r" (prev_mask) \
	35	: "cc", cl); \
	36	\
	37	return prev >> bitoff; \
	38	}
	39
	40	#define CMPXCHG_GEN(type, sfx, br, br2, cl) \
	41	static inline \
da58b23c	42	u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \
d0563a12 PX	43	{ \
	44	unsigned int prev, prev_mask, tmp, bitoff, off; \
	45	\
	46	off = (unsigned long)p % sizeof(u32); \
	47	bitoff = BITOFF_CAL(sizeof(type), off); \
	48	p -= off; \
	49	old <<= bitoff; \
	50	new <<= bitoff; \
	51	prev_mask = (u32)(type)-1 << bitoff; \
	52	\
	53	__asm__ __volatile__( \
	54	br \
	55	"1: lwarx %0,0,%3\n" \
	56	" and %1,%0,%6\n" \
	57	" cmpw 0,%1,%4\n" \
	58	" bne- 2f\n" \
	59	" andc %1,%0,%6\n" \
	60	" or %1,%1,%5\n" \
d0563a12 PX	61	" stwcx. %1,0,%3\n" \
	62	" bne- 1b\n" \
	63	br2 \
	64	"\n" \
	65	"2:" \
	66	: "=&r" (prev), "=&r" (tmp), "+m" ((u32)p) \
	67	: "r" (p), "r" (old), "r" (new), "r" (prev_mask) \
	68	: "cc", cl); \
	69	\
	70	return prev >> bitoff; \
	71	}
	72
ae3a197e DH	73	/*
	74	* Atomic exchange
	75	*
26760fc1	76	* Changes the memory location '*p' to be val and returns
ae3a197e DH	77	* the previous value stored there.
ae3a197e DH	78	*/
26760fc1	79
b86cf14f	80	#ifndef CONFIG_PPC_HAS_LBARX_LHARX
d0563a12 PX	81	XCHG_GEN(u8, _local, "memory");
	82	XCHG_GEN(u8, _relaxed, "cc");
	83	XCHG_GEN(u16, _local, "memory");
	84	XCHG_GEN(u16, _relaxed, "cc");
b86cf14f NP	85	#else
	86	static __always_inline unsigned long
	87	__xchg_u8_local(volatile void *p, unsigned long val)
	88	{
	89	unsigned long prev;
	90
	91	__asm__ __volatile__(
	92	"1: lbarx %0,0,%2 # __xchg_u8_local\n"
	93	" stbcx. %3,0,%2 \n"
	94	" bne- 1b"
	95	: "=&r" (prev), "+m" ((volatile unsigned char )p)
	96	: "r" (p), "r" (val)
	97	: "cc", "memory");
	98
	99	return prev;
	100	}
	101
	102	static __always_inline unsigned long
	103	__xchg_u8_relaxed(u8 *p, unsigned long val)
	104	{
	105	unsigned long prev;
	106
	107	__asm__ __volatile__(
	108	"1: lbarx %0,0,%2 # __xchg_u8_relaxed\n"
	109	" stbcx. %3,0,%2\n"
	110	" bne- 1b"
	111	: "=&r" (prev), "+m" (*p)
	112	: "r" (p), "r" (val)
	113	: "cc");
	114
	115	return prev;
	116	}
	117
	118	static __always_inline unsigned long
	119	__xchg_u16_local(volatile void *p, unsigned long val)
	120	{
	121	unsigned long prev;
	122
	123	__asm__ __volatile__(
	124	"1: lharx %0,0,%2 # __xchg_u16_local\n"
	125	" sthcx. %3,0,%2\n"
	126	" bne- 1b"
	127	: "=&r" (prev), "+m" ((volatile unsigned short )p)
	128	: "r" (p), "r" (val)
	129	: "cc", "memory");
	130
	131	return prev;
	132	}
	133
	134	static __always_inline unsigned long
	135	__xchg_u16_relaxed(u16 *p, unsigned long val)
	136	{
	137	unsigned long prev;
	138
	139	__asm__ __volatile__(
	140	"1: lharx %0,0,%2 # __xchg_u16_relaxed\n"
	141	" sthcx. %3,0,%2\n"
	142	" bne- 1b"
	143	: "=&r" (prev), "+m" (*p)
	144	: "r" (p), "r" (val)
	145	: "cc");
	146
	147	return prev;
	148	}
149	#endif
d0563a12	150
ae3a197e	151	static __always_inline unsigned long
26760fc1	152	__xchg_u32_local(volatile void *p, unsigned long val)
ae3a197e DH	153	{
	154	unsigned long prev;
	155
	156	__asm__ __volatile__(
ae3a197e	157	"1: lwarx %0,0,%2 \n"
ae3a197e DH	158	" stwcx. %3,0,%2 \n\
ae3a197e DH	159	bne- 1b"
ae3a197e DH	160	: "=&r" (prev), "+m" ((volatile unsigned int )p)
	161	: "r" (p), "r" (val)
	162	: "cc", "memory");
	163
	164	return prev;
	165	}
	166
ae3a197e	167	static __always_inline unsigned long
26760fc1	168	__xchg_u32_relaxed(u32 *p, unsigned long val)
ae3a197e DH	169	{
	170	unsigned long prev;
	171
	172	__asm__ __volatile__(
26760fc1	173	"1: lwarx %0,0,%2\n"
26760fc1 BF	174	" stwcx. %3,0,%2\n"
	175	" bne- 1b"
	176	: "=&r" (prev), "+m" (*p)
ae3a197e	177	: "r" (p), "r" (val)
26760fc1	178	: "cc");
ae3a197e DH	179
	180	return prev;
	181	}
	182
	183	#ifdef CONFIG_PPC64
	184	static __always_inline unsigned long
26760fc1	185	__xchg_u64_local(volatile void *p, unsigned long val)
ae3a197e DH	186	{
	187	unsigned long prev;
	188
	189	__asm__ __volatile__(
ae3a197e	190	"1: ldarx %0,0,%2 \n"
ae3a197e DH	191	" stdcx. %3,0,%2 \n\
ae3a197e DH	192	bne- 1b"
ae3a197e DH	193	: "=&r" (prev), "+m" ((volatile unsigned long )p)
	194	: "r" (p), "r" (val)
	195	: "cc", "memory");
	196
	197	return prev;
	198	}
	199
	200	static __always_inline unsigned long
26760fc1	201	__xchg_u64_relaxed(u64 *p, unsigned long val)
ae3a197e DH	202	{
	203	unsigned long prev;
	204
	205	__asm__ __volatile__(
26760fc1	206	"1: ldarx %0,0,%2\n"
26760fc1 BF	207	" stdcx. %3,0,%2\n"
	208	" bne- 1b"
	209	: "=&r" (prev), "+m" (*p)
ae3a197e	210	: "r" (p), "r" (val)
26760fc1	211	: "cc");
ae3a197e DH	212
	213	return prev;
	214	}
	215	#endif
	216
ae3a197e	217	static __always_inline unsigned long
d0563a12	218	__xchg_local(void *ptr, unsigned long x, unsigned int size)
ae3a197e DH	219	{
ae3a197e DH	220	switch (size) {
d0563a12 PX	221	case 1:
	222	return __xchg_u8_local(ptr, x);
	223	case 2:
	224	return __xchg_u16_local(ptr, x);
ae3a197e	225	case 4:
26760fc1	226	return __xchg_u32_local(ptr, x);
ae3a197e DH	227	#ifdef CONFIG_PPC64
ae3a197e DH	228	case 8:
26760fc1	229	return __xchg_u64_local(ptr, x);
ae3a197e DH	230	#endif
ae3a197e DH	231	}
06855063	232	BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
ae3a197e DH	233	return x;
	234	}
	235
	236	static __always_inline unsigned long
26760fc1	237	__xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
ae3a197e DH	238	{
ae3a197e DH	239	switch (size) {
d0563a12 PX	240	case 1:
	241	return __xchg_u8_relaxed(ptr, x);
	242	case 2:
	243	return __xchg_u16_relaxed(ptr, x);
ae3a197e	244	case 4:
26760fc1	245	return __xchg_u32_relaxed(ptr, x);
ae3a197e DH	246	#ifdef CONFIG_PPC64
ae3a197e DH	247	case 8:
26760fc1	248	return __xchg_u64_relaxed(ptr, x);
ae3a197e DH	249	#endif
ae3a197e DH	250	}
06855063	251	BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_relaxed");
ae3a197e DH	252	return x;
ae3a197e DH	253	}
9eaa8293	254	#define arch_xchg_local(ptr,x) \
ae3a197e DH	255	({ \
	256	__typeof__(*(ptr)) _x_ = (x); \
	257	(__typeof__(*(ptr))) __xchg_local((ptr), \
	258	(unsigned long)_x_, sizeof(*(ptr))); \
	259	})
	260
9eaa8293	261	#define arch_xchg_relaxed(ptr, x) \
26760fc1 BF	262	({ \
	263	__typeof__(*(ptr)) _x_ = (x); \
	264	(__typeof__(*(ptr))) __xchg_relaxed((ptr), \
	265	(unsigned long)_x_, sizeof(*(ptr))); \
	266	})
b86cf14f	267
ae3a197e DH	268	/*
	269	* Compare and exchange - if *p == old, set it to new,
	270	* and return the old value of *p.
	271	*/
b86cf14f	272	#ifndef CONFIG_PPC_HAS_LBARX_LHARX
d0563a12 PX	273	CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
	274	CMPXCHG_GEN(u8, _local, , , "memory");
	275	CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
	276	CMPXCHG_GEN(u8, _relaxed, , , "cc");
	277	CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
	278	CMPXCHG_GEN(u16, _local, , , "memory");
	279	CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
	280	CMPXCHG_GEN(u16, _relaxed, , , "cc");
b86cf14f NP	281	#else
	282	static __always_inline unsigned long
	283	__cmpxchg_u8(volatile unsigned char *p, unsigned long old, unsigned long new)
	284	{
	285	unsigned int prev;
	286
	287	__asm__ __volatile__ (
	288	PPC_ATOMIC_ENTRY_BARRIER
	289	"1: lbarx %0,0,%2 # __cmpxchg_u8\n"
	290	" cmpw 0,%0,%3\n"
	291	" bne- 2f\n"
	292	" stbcx. %4,0,%2\n"
	293	" bne- 1b"
	294	PPC_ATOMIC_EXIT_BARRIER
	295	"\n\
	296	2:"
	297	: "=&r" (prev), "+m" (*p)
	298	: "r" (p), "r" (old), "r" (new)
	299	: "cc", "memory");
	300
	301	return prev;
	302	}
	303
	304	static __always_inline unsigned long
	305	__cmpxchg_u8_local(volatile unsigned char *p, unsigned long old,
	306	unsigned long new)
	307	{
	308	unsigned int prev;
	309
	310	__asm__ __volatile__ (
	311	"1: lbarx %0,0,%2 # __cmpxchg_u8_local\n"
	312	" cmpw 0,%0,%3\n"
	313	" bne- 2f\n"
	314	" stbcx. %4,0,%2\n"
	315	" bne- 1b\n"
	316	"2:"
	317	: "=&r" (prev), "+m" (*p)
	318	: "r" (p), "r" (old), "r" (new)
	319	: "cc", "memory");
	320
	321	return prev;
	322	}
	323
	324	static __always_inline unsigned long
	325	__cmpxchg_u8_relaxed(u8 *p, unsigned long old, unsigned long new)
	326	{
	327	unsigned long prev;
	328
	329	__asm__ __volatile__ (
	330	"1: lbarx %0,0,%2 # __cmpxchg_u8_relaxed\n"
	331	" cmpw 0,%0,%3\n"
	332	" bne- 2f\n"
	333	" stbcx. %4,0,%2\n"
	334	" bne- 1b\n"
	335	"2:"
	336	: "=&r" (prev), "+m" (*p)
	337	: "r" (p), "r" (old), "r" (new)
	338	: "cc");
	339
	340	return prev;
	341	}
	342
	343	static __always_inline unsigned long
	344	__cmpxchg_u8_acquire(u8 *p, unsigned long old, unsigned long new)
345	{
346	unsigned long prev;
347
348	__asm__ __volatile__ (
349	"1: lbarx %0,0,%2 # __cmpxchg_u8_acquire\n"
350	" cmpw 0,%0,%3\n"
351	" bne- 2f\n"
352	" stbcx. %4,0,%2\n"
353	" bne- 1b\n"
354	PPC_ACQUIRE_BARRIER
355	"2:"
356	: "=&r" (prev), "+m" (*p)
357	: "r" (p), "r" (old), "r" (new)
358	: "cc", "memory");
359
360	return prev;
361	}
362
363	static __always_inline unsigned long
364	__cmpxchg_u16(volatile unsigned short *p, unsigned long old, unsigned long new)
365	{
366	unsigned int prev;
367
368	__asm__ __volatile__ (
369	PPC_ATOMIC_ENTRY_BARRIER
370	"1: lharx %0,0,%2 # __cmpxchg_u16\n"
371	" cmpw 0,%0,%3\n"
372	" bne- 2f\n"
373	" sthcx. %4,0,%2\n"
374	" bne- 1b\n"
375	PPC_ATOMIC_EXIT_BARRIER
376	"2:"
377	: "=&r" (prev), "+m" (*p)
378	: "r" (p), "r" (old), "r" (new)
379	: "cc", "memory");
380
381	return prev;
382	}
383
384	static __always_inline unsigned long
385	__cmpxchg_u16_local(volatile unsigned short *p, unsigned long old,
386	unsigned long new)
387	{
388	unsigned int prev;
389
390	__asm__ __volatile__ (
391	"1: lharx %0,0,%2 # __cmpxchg_u16_local\n"
392	" cmpw 0,%0,%3\n"
393	" bne- 2f\n"
394	" sthcx. %4,0,%2\n"
395	" bne- 1b"
396	"2:"
397	: "=&r" (prev), "+m" (*p)
398	: "r" (p), "r" (old), "r" (new)
399	: "cc", "memory");
400
401	return prev;
402	}
403
404	static __always_inline unsigned long
405	__cmpxchg_u16_relaxed(u16 *p, unsigned long old, unsigned long new)
406	{
407	unsigned long prev;
408
409	__asm__ __volatile__ (
410	"1: lharx %0,0,%2 # __cmpxchg_u16_relaxed\n"
411	" cmpw 0,%0,%3\n"
412	" bne- 2f\n"
413	" sthcx. %4,0,%2\n"
414	" bne- 1b\n"
415	"2:"
416	: "=&r" (prev), "+m" (*p)
417	: "r" (p), "r" (old), "r" (new)
418	: "cc");
419
420	return prev;
421	}
422
423	static __always_inline unsigned long
424	__cmpxchg_u16_acquire(u16 *p, unsigned long old, unsigned long new)
425	{
426	unsigned long prev;
427
428	__asm__ __volatile__ (
429	"1: lharx %0,0,%2 # __cmpxchg_u16_acquire\n"
430	" cmpw 0,%0,%3\n"
431	" bne- 2f\n"
432	" sthcx. %4,0,%2\n"
433	" bne- 1b\n"
434	PPC_ACQUIRE_BARRIER
435	"2:"
436	: "=&r" (prev), "+m" (*p)
437	: "r" (p), "r" (old), "r" (new)
438	: "cc", "memory");
439
440	return prev;
441	}
442	#endif
d0563a12	443
ae3a197e DH	444	static __always_inline unsigned long
	445	__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
	446	{
	447	unsigned int prev;
	448
	449	__asm__ __volatile__ (
81d7a329	450	PPC_ATOMIC_ENTRY_BARRIER
ae3a197e DH	451	"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
	452	cmpw 0,%0,%3\n\
	453	bne- 2f\n"
ae3a197e DH	454	" stwcx. %4,0,%2\n\
ae3a197e DH	455	bne- 1b"
81d7a329	456	PPC_ATOMIC_EXIT_BARRIER
ae3a197e DH	457	"\n\
	458	2:"
	459	: "=&r" (prev), "+m" (*p)
	460	: "r" (p), "r" (old), "r" (new)
	461	: "cc", "memory");
	462
	463	return prev;
	464	}
	465
	466	static __always_inline unsigned long
	467	__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
	468	unsigned long new)
	469	{
	470	unsigned int prev;
	471
	472	__asm__ __volatile__ (
	473	"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
	474	cmpw 0,%0,%3\n\
	475	bne- 2f\n"
ae3a197e DH	476	" stwcx. %4,0,%2\n\
	477	bne- 1b"
	478	"\n\
	479	2:"
	480	: "=&r" (prev), "+m" (*p)
	481	: "r" (p), "r" (old), "r" (new)
	482	: "cc", "memory");
	483
	484	return prev;
	485	}
	486
56c08e6d BF	487	static __always_inline unsigned long
	488	__cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
	489	{
	490	unsigned long prev;
	491
	492	__asm__ __volatile__ (
	493	"1: lwarx %0,0,%2 # __cmpxchg_u32_relaxed\n"
	494	" cmpw 0,%0,%3\n"
	495	" bne- 2f\n"
56c08e6d BF	496	" stwcx. %4,0,%2\n"
	497	" bne- 1b\n"
	498	"2:"
	499	: "=&r" (prev), "+m" (*p)
	500	: "r" (p), "r" (old), "r" (new)
	501	: "cc");
	502
	503	return prev;
	504	}
	505
	506	/*
	507	* cmpxchg family don't have order guarantee if cmp part fails, therefore we
	508	* can avoid superfluous barriers if we use assembly code to implement
	509	* cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
	510	* cmpxchg_release() because that will result in putting a barrier in the
	511	* middle of a ll/sc loop, which is probably a bad idea. For example, this
	512	* might cause the conditional store more likely to fail.
	513	*/
	514	static __always_inline unsigned long
	515	__cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
	516	{
	517	unsigned long prev;
	518
	519	__asm__ __volatile__ (
	520	"1: lwarx %0,0,%2 # __cmpxchg_u32_acquire\n"
	521	" cmpw 0,%0,%3\n"
	522	" bne- 2f\n"
56c08e6d BF	523	" stwcx. %4,0,%2\n"
	524	" bne- 1b\n"
	525	PPC_ACQUIRE_BARRIER
	526	"\n"
	527	"2:"
	528	: "=&r" (prev), "+m" (*p)
	529	: "r" (p), "r" (old), "r" (new)
	530	: "cc", "memory");
	531
	532	return prev;
	533	}
	534
ae3a197e DH	535	#ifdef CONFIG_PPC64
	536	static __always_inline unsigned long
	537	__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
	538	{
	539	unsigned long prev;
	540
	541	__asm__ __volatile__ (
81d7a329	542	PPC_ATOMIC_ENTRY_BARRIER
ae3a197e DH	543	"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
	544	cmpd 0,%0,%3\n\
	545	bne- 2f\n\
	546	stdcx. %4,0,%2\n\
	547	bne- 1b"
81d7a329	548	PPC_ATOMIC_EXIT_BARRIER
ae3a197e DH	549	"\n\
	550	2:"
	551	: "=&r" (prev), "+m" (*p)
	552	: "r" (p), "r" (old), "r" (new)
	553	: "cc", "memory");
	554
	555	return prev;
	556	}
	557
	558	static __always_inline unsigned long
	559	__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
	560	unsigned long new)
	561	{
	562	unsigned long prev;
	563
	564	__asm__ __volatile__ (
	565	"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
	566	cmpd 0,%0,%3\n\
	567	bne- 2f\n\
	568	stdcx. %4,0,%2\n\
	569	bne- 1b"
	570	"\n\
	571	2:"
	572	: "=&r" (prev), "+m" (*p)
	573	: "r" (p), "r" (old), "r" (new)
	574	: "cc", "memory");
	575
	576	return prev;
	577	}
56c08e6d BF	578
	579	static __always_inline unsigned long
	580	__cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
	581	{
	582	unsigned long prev;
	583
	584	__asm__ __volatile__ (
	585	"1: ldarx %0,0,%2 # __cmpxchg_u64_relaxed\n"
	586	" cmpd 0,%0,%3\n"
	587	" bne- 2f\n"
	588	" stdcx. %4,0,%2\n"
	589	" bne- 1b\n"
	590	"2:"
	591	: "=&r" (prev), "+m" (*p)
	592	: "r" (p), "r" (old), "r" (new)
	593	: "cc");
	594
	595	return prev;
	596	}
	597
	598	static __always_inline unsigned long
	599	__cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
	600	{
	601	unsigned long prev;
	602
	603	__asm__ __volatile__ (
	604	"1: ldarx %0,0,%2 # __cmpxchg_u64_acquire\n"
	605	" cmpd 0,%0,%3\n"
	606	" bne- 2f\n"
	607	" stdcx. %4,0,%2\n"
	608	" bne- 1b\n"
	609	PPC_ACQUIRE_BARRIER
	610	"\n"
	611	"2:"
	612	: "=&r" (prev), "+m" (*p)
	613	: "r" (p), "r" (old), "r" (new)
	614	: "cc", "memory");
	615
	616	return prev;
	617	}
ae3a197e DH	618	#endif
ae3a197e DH	619
ae3a197e	620	static __always_inline unsigned long
da58b23c	621	__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
ae3a197e DH	622	unsigned int size)
	623	{
	624	switch (size) {
d0563a12 PX	625	case 1:
	626	return __cmpxchg_u8(ptr, old, new);
	627	case 2:
	628	return __cmpxchg_u16(ptr, old, new);
ae3a197e DH	629	case 4:
	630	return __cmpxchg_u32(ptr, old, new);
	631	#ifdef CONFIG_PPC64
	632	case 8:
	633	return __cmpxchg_u64(ptr, old, new);
	634	#endif
	635	}
10d8b148	636	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
ae3a197e DH	637	return old;
	638	}
	639
	640	static __always_inline unsigned long
d0563a12	641	__cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
ae3a197e DH	642	unsigned int size)
	643	{
	644	switch (size) {
d0563a12 PX	645	case 1:
	646	return __cmpxchg_u8_local(ptr, old, new);
	647	case 2:
	648	return __cmpxchg_u16_local(ptr, old, new);
ae3a197e DH	649	case 4:
	650	return __cmpxchg_u32_local(ptr, old, new);
	651	#ifdef CONFIG_PPC64
	652	case 8:
	653	return __cmpxchg_u64_local(ptr, old, new);
	654	#endif
	655	}
10d8b148	656	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
ae3a197e DH	657	return old;
	658	}
	659
56c08e6d BF	660	static __always_inline unsigned long
	661	__cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
	662	unsigned int size)
	663	{
	664	switch (size) {
d0563a12 PX	665	case 1:
	666	return __cmpxchg_u8_relaxed(ptr, old, new);
	667	case 2:
	668	return __cmpxchg_u16_relaxed(ptr, old, new);
56c08e6d BF	669	case 4:
	670	return __cmpxchg_u32_relaxed(ptr, old, new);
	671	#ifdef CONFIG_PPC64
	672	case 8:
	673	return __cmpxchg_u64_relaxed(ptr, old, new);
	674	#endif
	675	}
10d8b148	676	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
56c08e6d BF	677	return old;
	678	}
	679
	680	static __always_inline unsigned long
	681	__cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
	682	unsigned int size)
	683	{
	684	switch (size) {
d0563a12 PX	685	case 1:
	686	return __cmpxchg_u8_acquire(ptr, old, new);
	687	case 2:
	688	return __cmpxchg_u16_acquire(ptr, old, new);
56c08e6d BF	689	case 4:
	690	return __cmpxchg_u32_acquire(ptr, old, new);
	691	#ifdef CONFIG_PPC64
	692	case 8:
	693	return __cmpxchg_u64_acquire(ptr, old, new);
	694	#endif
	695	}
10d8b148	696	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
56c08e6d BF	697	return old;
56c08e6d BF	698	}
9eaa8293	699	#define arch_cmpxchg(ptr, o, n) \
ae3a197e DH	700	({ \
	701	__typeof__(*(ptr)) _o_ = (o); \
	702	__typeof__(*(ptr)) _n_ = (n); \
	703	(__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
	704	(unsigned long)_n_, sizeof(*(ptr))); \
	705	})
	706
	707
9eaa8293	708	#define arch_cmpxchg_local(ptr, o, n) \
ae3a197e DH	709	({ \
	710	__typeof__(*(ptr)) _o_ = (o); \
	711	__typeof__(*(ptr)) _n_ = (n); \
	712	(__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \
	713	(unsigned long)_n_, sizeof(*(ptr))); \
	714	})
	715
9eaa8293	716	#define arch_cmpxchg_relaxed(ptr, o, n) \
56c08e6d BF	717	({ \
	718	__typeof__(*(ptr)) _o_ = (o); \
	719	__typeof__(*(ptr)) _n_ = (n); \
	720	(__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
	721	(unsigned long)_o_, (unsigned long)_n_, \
	722	sizeof(*(ptr))); \
	723	})
	724
9eaa8293	725	#define arch_cmpxchg_acquire(ptr, o, n) \
56c08e6d BF	726	({ \
	727	__typeof__(*(ptr)) _o_ = (o); \
	728	__typeof__(*(ptr)) _n_ = (n); \
	729	(__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
	730	(unsigned long)_o_, (unsigned long)_n_, \
	731	sizeof(*(ptr))); \
	732	})
ae3a197e	733	#ifdef CONFIG_PPC64
9eaa8293	734	#define arch_cmpxchg64(ptr, o, n) \
ae3a197e DH	735	({ \
ae3a197e DH	736	BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
9eaa8293	737	arch_cmpxchg((ptr), (o), (n)); \
ae3a197e	738	})
9eaa8293	739	#define arch_cmpxchg64_local(ptr, o, n) \
ae3a197e DH	740	({ \
ae3a197e DH	741	BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
9eaa8293	742	arch_cmpxchg_local((ptr), (o), (n)); \
ae3a197e	743	})
9eaa8293	744	#define arch_cmpxchg64_relaxed(ptr, o, n) \
56c08e6d BF	745	({ \
56c08e6d BF	746	BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
9eaa8293	747	arch_cmpxchg_relaxed((ptr), (o), (n)); \
56c08e6d	748	})
9eaa8293	749	#define arch_cmpxchg64_acquire(ptr, o, n) \
56c08e6d BF	750	({ \
56c08e6d BF	751	BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
9eaa8293	752	arch_cmpxchg_acquire((ptr), (o), (n)); \
56c08e6d	753	})
ae3a197e DH	754	#else
ae3a197e DH	755	#include <asm-generic/cmpxchg-local.h>
9eaa8293	756	#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
ae3a197e DH	757	#endif
	758
	759	#endif /* __KERNEL__ */
	760	#endif /* _ASM_POWERPC_CMPXCHG_H_ */