[linux-block.git] / arch / riscv / include / asm / atomic.h

/*
 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
 * Copyright (C) 2012 Regents of the University of California
 * Copyright (C) 2017 SiFive
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public Licence
 * as published by the Free Software Foundation; either version
 * 2 of the Licence, or (at your option) any later version.
 */

#ifndef _ASM_RISCV_ATOMIC_H
#define _ASM_RISCV_ATOMIC_H

#ifdef CONFIG_GENERIC_ATOMIC64
# include <asm-generic/atomic64.h>
#else
# if (__riscv_xlen < 64)
#  error "64-bit atomics require XLEN to be at least 64"
# endif
#endif

#include <asm/cmpxchg.h>
#include <asm/barrier.h>

#define ATOMIC_INIT(i)	{ (i) }
static __always_inline int atomic_read(const atomic_t *v)
{
	return READ_ONCE(v->counter);
}
static __always_inline void atomic_set(atomic_t *v, int i)
{
	WRITE_ONCE(v->counter, i);
}

#ifndef CONFIG_GENERIC_ATOMIC64
#define ATOMIC64_INIT(i) { (i) }
static __always_inline long atomic64_read(const atomic64_t *v)
{
	return READ_ONCE(v->counter);
}
static __always_inline void atomic64_set(atomic64_t *v, long i)
{
	WRITE_ONCE(v->counter, i);
}
#endif

/*
 * First, the atomic ops that have no ordering constraints and therefor don't
 * have the AQ or RL bits set.  These don't return anything, so there's only
 * one version to worry about.
 */
#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix)				\
static __always_inline void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v)	\
{											\
	__asm__ __volatile__ (								\
		"amo" #asm_op "." #asm_type " zero, %1, %0"				\
		: "+A" (v->counter)							\
		: "r" (I)								\
		: "memory");								\
}

#ifdef CONFIG_GENERIC_ATOMIC64
#define ATOMIC_OPS(op, asm_op, I)			\
        ATOMIC_OP (op, asm_op, I, w,  int,   )
#else
#define ATOMIC_OPS(op, asm_op, I)			\
        ATOMIC_OP (op, asm_op, I, w,  int,   )	\
        ATOMIC_OP (op, asm_op, I, d, long, 64)
#endif

ATOMIC_OPS(add, add,  i)
ATOMIC_OPS(sub, add, -i)
ATOMIC_OPS(and, and,  i)
ATOMIC_OPS( or,  or,  i)
ATOMIC_OPS(xor, xor,  i)

#undef ATOMIC_OP
#undef ATOMIC_OPS

/*
 * Atomic ops that have ordered, relaxed, acquire, and relese variants.
 * There's two flavors of these: the arithmatic ops have both fetch and return
 * versions, while the logical ops only have fetch versions.
 */
#define ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, asm_type, c_type, prefix)				\
static __always_inline c_type atomic##prefix##_fetch_##op##c_or(c_type i, atomic##prefix##_t *v)	\
{													\
	register c_type ret;										\
	__asm__ __volatile__ (										\
		"amo" #asm_op "." #asm_type #asm_or " %1, %2, %0"					\
		: "+A" (v->counter), "=r" (ret)								\
		: "r" (I)										\
		: "memory");										\
	return ret;											\
}

#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, asm_type, c_type, prefix)			\
static __always_inline c_type atomic##prefix##_##op##_return##c_or(c_type i, atomic##prefix##_t *v)	\
{													\
        return atomic##prefix##_fetch_##op##c_or(i, v) c_op I;						\
}

#ifdef CONFIG_GENERIC_ATOMIC64
#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or)				\
        ATOMIC_FETCH_OP (op, asm_op,       I, asm_or, c_or, w,  int,   )	\
        ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w,  int,   )
#else
#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or)				\
        ATOMIC_FETCH_OP (op, asm_op,       I, asm_or, c_or, w,  int,   )	\
        ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w,  int,   )	\
        ATOMIC_FETCH_OP (op, asm_op,       I, asm_or, c_or, d, long, 64)	\
        ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, d, long, 64)
#endif

ATOMIC_OPS(add, add, +,  i,      , _relaxed)
ATOMIC_OPS(add, add, +,  i, .aq  , _acquire)
ATOMIC_OPS(add, add, +,  i, .rl  , _release)
ATOMIC_OPS(add, add, +,  i, .aqrl,         )

ATOMIC_OPS(sub, add, +, -i,      , _relaxed)
ATOMIC_OPS(sub, add, +, -i, .aq  , _acquire)
ATOMIC_OPS(sub, add, +, -i, .rl  , _release)
ATOMIC_OPS(sub, add, +, -i, .aqrl,         )

#undef ATOMIC_OPS

#ifdef CONFIG_GENERIC_ATOMIC64
#define ATOMIC_OPS(op, asm_op, I, asm_or, c_or)				\
        ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, w,  int,   )
#else
#define ATOMIC_OPS(op, asm_op, I, asm_or, c_or)				\
        ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, w,  int,   )	\
        ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, d, long, 64)
#endif

ATOMIC_OPS(and, and, i,      , _relaxed)
ATOMIC_OPS(and, and, i, .aq  , _acquire)
ATOMIC_OPS(and, and, i, .rl  , _release)
ATOMIC_OPS(and, and, i, .aqrl,         )

ATOMIC_OPS( or,  or, i,      , _relaxed)
ATOMIC_OPS( or,  or, i, .aq  , _acquire)
ATOMIC_OPS( or,  or, i, .rl  , _release)
ATOMIC_OPS( or,  or, i, .aqrl,         )

ATOMIC_OPS(xor, xor, i,      , _relaxed)
ATOMIC_OPS(xor, xor, i, .aq  , _acquire)
ATOMIC_OPS(xor, xor, i, .rl  , _release)
ATOMIC_OPS(xor, xor, i, .aqrl,         )

#undef ATOMIC_OPS

#undef ATOMIC_FETCH_OP
#undef ATOMIC_OP_RETURN

/*
 * The extra atomic operations that are constructed from one of the core
 * AMO-based operations above (aside from sub, which is easier to fit above).
 * These are required to perform a barrier, but they're OK this way because
 * atomic_*_return is also required to perform a barrier.
 */
#define ATOMIC_OP(op, func_op, comp_op, I, c_type, prefix)			\
static __always_inline bool atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
{										\
	return atomic##prefix##_##func_op##_return(i, v) comp_op I;		\
}

#ifdef CONFIG_GENERIC_ATOMIC64
#define ATOMIC_OPS(op, func_op, comp_op, I)			\
        ATOMIC_OP (op, func_op, comp_op, I,  int,   )
#else
#define ATOMIC_OPS(op, func_op, comp_op, I)			\
        ATOMIC_OP (op, func_op, comp_op, I,  int,   )		\
        ATOMIC_OP (op, func_op, comp_op, I, long, 64)
#endif

ATOMIC_OPS(add_and_test, add, ==, 0)
ATOMIC_OPS(sub_and_test, sub, ==, 0)
ATOMIC_OPS(add_negative, add,  <, 0)

#undef ATOMIC_OP
#undef ATOMIC_OPS

#define ATOMIC_OP(op, func_op, I, c_type, prefix)				\
static __always_inline void atomic##prefix##_##op(atomic##prefix##_t *v)	\
{										\
	atomic##prefix##_##func_op(I, v);					\
}

#define ATOMIC_FETCH_OP(op, func_op, I, c_type, prefix)					\
static __always_inline c_type atomic##prefix##_fetch_##op(atomic##prefix##_t *v)	\
{											\
	return atomic##prefix##_fetch_##func_op(I, v);					\
}

#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, c_type, prefix)				\
static __always_inline c_type atomic##prefix##_##op##_return(atomic##prefix##_t *v)	\
{											\
        return atomic##prefix##_fetch_##op(v) c_op I;					\
}

#ifdef CONFIG_GENERIC_ATOMIC64
#define ATOMIC_OPS(op, asm_op, c_op, I)						\
        ATOMIC_OP       (op, asm_op,       I,  int,   )				\
        ATOMIC_FETCH_OP (op, asm_op,       I,  int,   )				\
        ATOMIC_OP_RETURN(op, asm_op, c_op, I,  int,   )
#else
#define ATOMIC_OPS(op, asm_op, c_op, I)						\
        ATOMIC_OP       (op, asm_op,       I,  int,   )				\
        ATOMIC_FETCH_OP (op, asm_op,       I,  int,   )				\
        ATOMIC_OP_RETURN(op, asm_op, c_op, I,  int,   )				\
        ATOMIC_OP       (op, asm_op,       I, long, 64)				\
        ATOMIC_FETCH_OP (op, asm_op,       I, long, 64)				\
        ATOMIC_OP_RETURN(op, asm_op, c_op, I, long, 64)
#endif

ATOMIC_OPS(inc, add, +,  1)
ATOMIC_OPS(dec, add, +, -1)

#undef ATOMIC_OPS
#undef ATOMIC_OP
#undef ATOMIC_FETCH_OP
#undef ATOMIC_OP_RETURN

#define ATOMIC_OP(op, func_op, comp_op, I, prefix)				\
static __always_inline bool atomic##prefix##_##op(atomic##prefix##_t *v)	\
{										\
	return atomic##prefix##_##func_op##_return(v) comp_op I;		\
}

ATOMIC_OP(inc_and_test, inc, ==, 0,   )
ATOMIC_OP(dec_and_test, dec, ==, 0,   )
#ifndef CONFIG_GENERIC_ATOMIC64
ATOMIC_OP(inc_and_test, inc, ==, 0, 64)
ATOMIC_OP(dec_and_test, dec, ==, 0, 64)
#endif

#undef ATOMIC_OP

/* This is required to provide a barrier on success. */
static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u)
{
       int prev, rc;

	__asm__ __volatile__ (
		"0:\n\t"
		"lr.w.aqrl  %[p],  %[c]\n\t"
		"beq        %[p],  %[u], 1f\n\t"
		"add       %[rc],  %[p], %[a]\n\t"
		"sc.w.aqrl %[rc], %[rc], %[c]\n\t"
		"bnez      %[rc], 0b\n\t"
		"1:"
		: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
		: [a]"r" (a), [u]"r" (u)
		: "memory");
	return prev;
}

#ifndef CONFIG_GENERIC_ATOMIC64
static __always_inline long __atomic64_add_unless(atomic64_t *v, long a, long u)
{
       long prev, rc;

	__asm__ __volatile__ (
		"0:\n\t"
		"lr.d.aqrl  %[p],  %[c]\n\t"
		"beq        %[p],  %[u], 1f\n\t"
		"add       %[rc],  %[p], %[a]\n\t"
		"sc.d.aqrl %[rc], %[rc], %[c]\n\t"
		"bnez      %[rc], 0b\n\t"
		"1:"
		: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
		: [a]"r" (a), [u]"r" (u)
		: "memory");
	return prev;
}

static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u)
{
	return __atomic64_add_unless(v, a, u) != u;
}
#endif

/*
 * The extra atomic operations that are constructed from one of the core
 * LR/SC-based operations above.
 */
static __always_inline int atomic_inc_not_zero(atomic_t *v)
{
        return __atomic_add_unless(v, 1, 0);
}

#ifndef CONFIG_GENERIC_ATOMIC64
static __always_inline long atomic64_inc_not_zero(atomic64_t *v)
{
        return atomic64_add_unless(v, 1, 0);
}
#endif

/*
 * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
 * {cmp,}xchg and the operations that return, so they need a barrier.
 */
/*
 * FIXME: atomic_cmpxchg_{acquire,release,relaxed} are all implemented by
 * assigning the same barrier to both the LR and SC operations, but that might
 * not make any sense.  We're waiting on a memory model specification to
 * determine exactly what the right thing to do is here.
 */
#define ATOMIC_OP(c_t, prefix, c_or, size, asm_or)						\
static __always_inline c_t atomic##prefix##_cmpxchg##c_or(atomic##prefix##_t *v, c_t o, c_t n) 	\
{												\
	return __cmpxchg(&(v->counter), o, n, size, asm_or, asm_or);				\
}												\
static __always_inline c_t atomic##prefix##_xchg##c_or(atomic##prefix##_t *v, c_t n) 		\
{												\
	return __xchg(n, &(v->counter), size, asm_or);						\
}

#ifdef CONFIG_GENERIC_ATOMIC64
#define ATOMIC_OPS(c_or, asm_or)			\
	ATOMIC_OP( int,   , c_or, 4, asm_or)
#else
#define ATOMIC_OPS(c_or, asm_or)			\
	ATOMIC_OP( int,   , c_or, 4, asm_or)		\
	ATOMIC_OP(long, 64, c_or, 8, asm_or)
#endif

ATOMIC_OPS(        , .aqrl)
ATOMIC_OPS(_acquire,   .aq)
ATOMIC_OPS(_release,   .rl)
ATOMIC_OPS(_relaxed,      )

#undef ATOMIC_OPS
#undef ATOMIC_OP

static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset)
{
       int prev, rc;

	__asm__ __volatile__ (
		"0:\n\t"
		"lr.w.aqrl  %[p],  %[c]\n\t"
		"sub       %[rc],  %[p], %[o]\n\t"
		"bltz      %[rc],    1f\n\t"
		"sc.w.aqrl %[rc], %[rc], %[c]\n\t"
		"bnez      %[rc],    0b\n\t"
		"1:"
		: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
		: [o]"r" (offset)
		: "memory");
	return prev - offset;
}

#define atomic_dec_if_positive(v)	atomic_sub_if_positive(v, 1)

#ifndef CONFIG_GENERIC_ATOMIC64
static __always_inline long atomic64_sub_if_positive(atomic64_t *v, int offset)
{
       long prev, rc;

	__asm__ __volatile__ (
		"0:\n\t"
		"lr.d.aqrl  %[p],  %[c]\n\t"
		"sub       %[rc],  %[p], %[o]\n\t"
		"bltz      %[rc],    1f\n\t"
		"sc.d.aqrl %[rc], %[rc], %[c]\n\t"
		"bnez      %[rc],    0b\n\t"
		"1:"
		: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
		: [o]"r" (offset)
		: "memory");
	return prev - offset;
}

#define atomic64_dec_if_positive(v)	atomic64_sub_if_positive(v, 1)
#endif

#endif /* _ASM_RISCV_ATOMIC_H */
Commit	Line	Data
fab957c1 PD	1	/*
	2	* Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
	3	* Copyright (C) 2012 Regents of the University of California
	4	* Copyright (C) 2017 SiFive
	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public Licence
	8	* as published by the Free Software Foundation; either version
	9	* 2 of the Licence, or (at your option) any later version.
	10	*/
	11
	12	#ifndef _ASM_RISCV_ATOMIC_H
	13	#define _ASM_RISCV_ATOMIC_H
	14
	15	#ifdef CONFIG_GENERIC_ATOMIC64
	16	# include <asm-generic/atomic64.h>
	17	#else
	18	# if (__riscv_xlen < 64)
	19	# error "64-bit atomics require XLEN to be at least 64"
	20	# endif
	21	#endif
	22
	23	#include <asm/cmpxchg.h>
	24	#include <asm/barrier.h>
	25
	26	#define ATOMIC_INIT(i) { (i) }
	27	static __always_inline int atomic_read(const atomic_t *v)
	28	{
	29	return READ_ONCE(v->counter);
	30	}
	31	static __always_inline void atomic_set(atomic_t *v, int i)
	32	{
	33	WRITE_ONCE(v->counter, i);
	34	}
	35
	36	#ifndef CONFIG_GENERIC_ATOMIC64
	37	#define ATOMIC64_INIT(i) { (i) }
	38	static __always_inline long atomic64_read(const atomic64_t *v)
	39	{
	40	return READ_ONCE(v->counter);
	41	}
	42	static __always_inline void atomic64_set(atomic64_t *v, long i)
	43	{
	44	WRITE_ONCE(v->counter, i);
	45	}
	46	#endif
	47
	48	/*
	49	* First, the atomic ops that have no ordering constraints and therefor don't
	50	* have the AQ or RL bits set. These don't return anything, so there's only
	51	* one version to worry about.
	52	*/
4650d02a PD	53	#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \
	54	static __always_inline void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
	55	{ \
	56	__asm__ __volatile__ ( \
	57	"amo" #asm_op "." #asm_type " zero, %1, %0" \
	58	: "+A" (v->counter) \
	59	: "r" (I) \
	60	: "memory"); \
fab957c1 PD	61	}
	62
	63	#ifdef CONFIG_GENERIC_ATOMIC64
4650d02a PD	64	#define ATOMIC_OPS(op, asm_op, I) \
4650d02a PD	65	ATOMIC_OP (op, asm_op, I, w, int, )
fab957c1	66	#else
4650d02a PD	67	#define ATOMIC_OPS(op, asm_op, I) \
	68	ATOMIC_OP (op, asm_op, I, w, int, ) \
	69	ATOMIC_OP (op, asm_op, I, d, long, 64)
fab957c1 PD	70	#endif
fab957c1 PD	71
4650d02a PD	72	ATOMIC_OPS(add, add, i)
	73	ATOMIC_OPS(sub, add, -i)
	74	ATOMIC_OPS(and, and, i)
	75	ATOMIC_OPS( or, or, i)
	76	ATOMIC_OPS(xor, xor, i)
fab957c1 PD	77
	78	#undef ATOMIC_OP
	79	#undef ATOMIC_OPS
	80
	81	/*
	82	* Atomic ops that have ordered, relaxed, acquire, and relese variants.
	83	* There's two flavors of these: the arithmatic ops have both fetch and return
	84	* versions, while the logical ops only have fetch versions.
	85	*/
4650d02a	86	#define ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, asm_type, c_type, prefix) \
fab957c1 PD	87	static __always_inline c_type atomic##prefix##_fetch_##op##c_or(c_type i, atomic##prefix##_t *v) \
	88	{ \
	89	register c_type ret; \
	90	__asm__ __volatile__ ( \
	91	"amo" #asm_op "." #asm_type #asm_or " %1, %2, %0" \
	92	: "+A" (v->counter), "=r" (ret) \
	93	: "r" (I) \
	94	: "memory"); \
	95	return ret; \
	96	}
	97
	98	#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, asm_type, c_type, prefix) \
	99	static __always_inline c_type atomic##prefix##_##op##_return##c_or(c_type i, atomic##prefix##_t *v) \
	100	{ \
	101	return atomic##prefix##_fetch_##op##c_or(i, v) c_op I; \
	102	}
	103
	104	#ifdef CONFIG_GENERIC_ATOMIC64
	105	#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or) \
4650d02a	106	ATOMIC_FETCH_OP (op, asm_op, I, asm_or, c_or, w, int, ) \
fab957c1 PD	107	ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w, int, )
	108	#else
	109	#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or) \
4650d02a	110	ATOMIC_FETCH_OP (op, asm_op, I, asm_or, c_or, w, int, ) \
fab957c1	111	ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w, int, ) \
4650d02a	112	ATOMIC_FETCH_OP (op, asm_op, I, asm_or, c_or, d, long, 64) \
fab957c1 PD	113	ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, d, long, 64)
	114	#endif
	115
	116	ATOMIC_OPS(add, add, +, i, , _relaxed)
	117	ATOMIC_OPS(add, add, +, i, .aq , _acquire)
	118	ATOMIC_OPS(add, add, +, i, .rl , _release)
	119	ATOMIC_OPS(add, add, +, i, .aqrl, )
	120
	121	ATOMIC_OPS(sub, add, +, -i, , _relaxed)
	122	ATOMIC_OPS(sub, add, +, -i, .aq , _acquire)
	123	ATOMIC_OPS(sub, add, +, -i, .rl , _release)
	124	ATOMIC_OPS(sub, add, +, -i, .aqrl, )
	125
	126	#undef ATOMIC_OPS
	127
	128	#ifdef CONFIG_GENERIC_ATOMIC64
4650d02a PD	129	#define ATOMIC_OPS(op, asm_op, I, asm_or, c_or) \
4650d02a PD	130	ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, w, int, )
fab957c1	131	#else
4650d02a PD	132	#define ATOMIC_OPS(op, asm_op, I, asm_or, c_or) \
	133	ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, w, int, ) \
	134	ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, d, long, 64)
fab957c1 PD	135	#endif
fab957c1 PD	136
4650d02a PD	137	ATOMIC_OPS(and, and, i, , _relaxed)
	138	ATOMIC_OPS(and, and, i, .aq , _acquire)
	139	ATOMIC_OPS(and, and, i, .rl , _release)
	140	ATOMIC_OPS(and, and, i, .aqrl, )
fab957c1	141
4650d02a PD	142	ATOMIC_OPS( or, or, i, , _relaxed)
	143	ATOMIC_OPS( or, or, i, .aq , _acquire)
	144	ATOMIC_OPS( or, or, i, .rl , _release)
	145	ATOMIC_OPS( or, or, i, .aqrl, )
fab957c1	146
4650d02a PD	147	ATOMIC_OPS(xor, xor, i, , _relaxed)
	148	ATOMIC_OPS(xor, xor, i, .aq , _acquire)
	149	ATOMIC_OPS(xor, xor, i, .rl , _release)
	150	ATOMIC_OPS(xor, xor, i, .aqrl, )
fab957c1 PD	151
	152	#undef ATOMIC_OPS
	153
	154	#undef ATOMIC_FETCH_OP
	155	#undef ATOMIC_OP_RETURN
	156
	157	/*
	158	* The extra atomic operations that are constructed from one of the core
	159	* AMO-based operations above (aside from sub, which is easier to fit above).
	160	* These are required to perform a barrier, but they're OK this way because
	161	* atomic_*_return is also required to perform a barrier.
	162	*/
	163	#define ATOMIC_OP(op, func_op, comp_op, I, c_type, prefix) \
	164	static __always_inline bool atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
	165	{ \
	166	return atomic##prefix##_##func_op##_return(i, v) comp_op I; \
	167	}
	168
	169	#ifdef CONFIG_GENERIC_ATOMIC64
	170	#define ATOMIC_OPS(op, func_op, comp_op, I) \
	171	ATOMIC_OP (op, func_op, comp_op, I, int, )
	172	#else
	173	#define ATOMIC_OPS(op, func_op, comp_op, I) \
	174	ATOMIC_OP (op, func_op, comp_op, I, int, ) \
	175	ATOMIC_OP (op, func_op, comp_op, I, long, 64)
	176	#endif
	177
	178	ATOMIC_OPS(add_and_test, add, ==, 0)
	179	ATOMIC_OPS(sub_and_test, sub, ==, 0)
	180	ATOMIC_OPS(add_negative, add, <, 0)
	181
	182	#undef ATOMIC_OP
	183	#undef ATOMIC_OPS
	184
4650d02a	185	#define ATOMIC_OP(op, func_op, I, c_type, prefix) \
fab957c1 PD	186	static __always_inline void atomic##prefix##_##op(atomic##prefix##_t *v) \
	187	{ \
	188	atomic##prefix##_##func_op(I, v); \
	189	}
	190
4650d02a	191	#define ATOMIC_FETCH_OP(op, func_op, I, c_type, prefix) \
fab957c1 PD	192	static __always_inline c_type atomic##prefix##_fetch_##op(atomic##prefix##_t *v) \
	193	{ \
	194	return atomic##prefix##_fetch_##func_op(I, v); \
	195	}
	196
	197	#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, c_type, prefix) \
	198	static __always_inline c_type atomic##prefix##_##op##_return(atomic##prefix##_t *v) \
	199	{ \
	200	return atomic##prefix##_fetch_##op(v) c_op I; \
	201	}
	202
	203	#ifdef CONFIG_GENERIC_ATOMIC64
	204	#define ATOMIC_OPS(op, asm_op, c_op, I) \
4650d02a PD	205	ATOMIC_OP (op, asm_op, I, int, ) \
4650d02a PD	206	ATOMIC_FETCH_OP (op, asm_op, I, int, ) \
fab957c1 PD	207	ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, )
	208	#else
	209	#define ATOMIC_OPS(op, asm_op, c_op, I) \
4650d02a PD	210	ATOMIC_OP (op, asm_op, I, int, ) \
4650d02a PD	211	ATOMIC_FETCH_OP (op, asm_op, I, int, ) \
fab957c1	212	ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, ) \
4650d02a PD	213	ATOMIC_OP (op, asm_op, I, long, 64) \
4650d02a PD	214	ATOMIC_FETCH_OP (op, asm_op, I, long, 64) \
fab957c1 PD	215	ATOMIC_OP_RETURN(op, asm_op, c_op, I, long, 64)
	216	#endif
	217
	218	ATOMIC_OPS(inc, add, +, 1)
	219	ATOMIC_OPS(dec, add, +, -1)
	220
	221	#undef ATOMIC_OPS
	222	#undef ATOMIC_OP
	223	#undef ATOMIC_FETCH_OP
	224	#undef ATOMIC_OP_RETURN
	225
	226	#define ATOMIC_OP(op, func_op, comp_op, I, prefix) \
	227	static __always_inline bool atomic##prefix##_##op(atomic##prefix##_t *v) \
	228	{ \
	229	return atomic##prefix##_##func_op##_return(v) comp_op I; \
	230	}
	231
	232	ATOMIC_OP(inc_and_test, inc, ==, 0, )
	233	ATOMIC_OP(dec_and_test, dec, ==, 0, )
	234	#ifndef CONFIG_GENERIC_ATOMIC64
	235	ATOMIC_OP(inc_and_test, inc, ==, 0, 64)
	236	ATOMIC_OP(dec_and_test, dec, ==, 0, 64)
	237	#endif
	238
	239	#undef ATOMIC_OP
	240
	241	/* This is required to provide a barrier on success. */
	242	static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u)
	243	{
	244	int prev, rc;
	245
	246	__asm__ __volatile__ (
	247	"0:\n\t"
	248	"lr.w.aqrl %[p], %[c]\n\t"
	249	"beq %[p], %[u], 1f\n\t"
	250	"add %[rc], %[p], %[a]\n\t"
	251	"sc.w.aqrl %[rc], %[rc], %[c]\n\t"
	252	"bnez %[rc], 0b\n\t"
	253	"1:"
	254	: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
	255	: [a]"r" (a), [u]"r" (u)
	256	: "memory");
	257	return prev;
	258	}
	259
	260	#ifndef CONFIG_GENERIC_ATOMIC64
	261	static __always_inline long __atomic64_add_unless(atomic64_t *v, long a, long u)
	262	{
	263	long prev, rc;
	264
	265	__asm__ __volatile__ (
	266	"0:\n\t"
	267	"lr.d.aqrl %[p], %[c]\n\t"
	268	"beq %[p], %[u], 1f\n\t"
	269	"add %[rc], %[p], %[a]\n\t"
	270	"sc.d.aqrl %[rc], %[rc], %[c]\n\t"
	271	"bnez %[rc], 0b\n\t"
	272	"1:"
	273	: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
	274	: [a]"r" (a), [u]"r" (u)
	275	: "memory");
	276	return prev;
	277	}
	278
279	static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u)
280	{
281	return __atomic64_add_unless(v, a, u) != u;
282	}
283	#endif
284
285	/*
286	* The extra atomic operations that are constructed from one of the core
287	* LR/SC-based operations above.
288	*/
289	static __always_inline int atomic_inc_not_zero(atomic_t *v)
290	{
291	return __atomic_add_unless(v, 1, 0);
292	}
293
294	#ifndef CONFIG_GENERIC_ATOMIC64
295	static __always_inline long atomic64_inc_not_zero(atomic64_t *v)
296	{
297	return atomic64_add_unless(v, 1, 0);
298	}
299	#endif
300
301	/*
302	* atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
8286d51a PD	303	* {cmp,}xchg and the operations that return, so they need a barrier.
	304	*/
	305	/*
	306	* FIXME: atomic_cmpxchg_{acquire,release,relaxed} are all implemented by
	307	* assigning the same barrier to both the LR and SC operations, but that might
	308	* not make any sense. We're waiting on a memory model specification to
	309	* determine exactly what the right thing to do is here.
fab957c1 PD	310	*/
	311	#define ATOMIC_OP(c_t, prefix, c_or, size, asm_or) \
	312	static __always_inline c_t atomic##prefix##_cmpxchg##c_or(atomic##prefix##_t *v, c_t o, c_t n) \
	313	{ \
	314	return __cmpxchg(&(v->counter), o, n, size, asm_or, asm_or); \
	315	} \
	316	static __always_inline c_t atomic##prefix##_xchg##c_or(atomic##prefix##_t *v, c_t n) \
	317	{ \
	318	return __xchg(n, &(v->counter), size, asm_or); \
	319	}
	320
	321	#ifdef CONFIG_GENERIC_ATOMIC64
	322	#define ATOMIC_OPS(c_or, asm_or) \
	323	ATOMIC_OP( int, , c_or, 4, asm_or)
	324	#else
	325	#define ATOMIC_OPS(c_or, asm_or) \
	326	ATOMIC_OP( int, , c_or, 4, asm_or) \
	327	ATOMIC_OP(long, 64, c_or, 8, asm_or)
	328	#endif
	329
	330	ATOMIC_OPS( , .aqrl)
	331	ATOMIC_OPS(_acquire, .aq)
	332	ATOMIC_OPS(_release, .rl)
	333	ATOMIC_OPS(_relaxed, )
	334
	335	#undef ATOMIC_OPS
	336	#undef ATOMIC_OP
	337
	338	static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset)
	339	{
	340	int prev, rc;
	341
	342	__asm__ __volatile__ (
	343	"0:\n\t"
	344	"lr.w.aqrl %[p], %[c]\n\t"
	345	"sub %[rc], %[p], %[o]\n\t"
	346	"bltz %[rc], 1f\n\t"
	347	"sc.w.aqrl %[rc], %[rc], %[c]\n\t"
	348	"bnez %[rc], 0b\n\t"
	349	"1:"
	350	: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
	351	: [o]"r" (offset)
	352	: "memory");
	353	return prev - offset;
	354	}
	355
	356	#define atomic_dec_if_positive(v) atomic_sub_if_positive(v, 1)
	357
	358	#ifndef CONFIG_GENERIC_ATOMIC64
	359	static __always_inline long atomic64_sub_if_positive(atomic64_t *v, int offset)
	360	{
	361	long prev, rc;
	362
	363	__asm__ __volatile__ (
	364	"0:\n\t"
	365	"lr.d.aqrl %[p], %[c]\n\t"
	366	"sub %[rc], %[p], %[o]\n\t"
	367	"bltz %[rc], 1f\n\t"
	368	"sc.d.aqrl %[rc], %[rc], %[c]\n\t"
	369	"bnez %[rc], 0b\n\t"
	370	"1:"
	371	: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
	372	: [o]"r" (offset)
	373	: "memory");
374	return prev - offset;
375	}
376
377	#define atomic64_dec_if_positive(v) atomic64_sub_if_positive(v, 1)
378	#endif
379
380	#endif /* _ASM_RISCV_ATOMIC_H */