[linux-2.6-block.git] / include / linux / atomic.h

/* Atomic operations usable in machine independent code */
#ifndef _LINUX_ATOMIC_H
#define _LINUX_ATOMIC_H
#include <asm/atomic.h>
#include <asm/barrier.h>

/*
 * Relaxed variants of xchg, cmpxchg and some atomic operations.
 *
 * We support four variants:
 *
 * - Fully ordered: The default implementation, no suffix required.
 * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
 * - Release: Provides RELEASE semantics, _release suffix.
 * - Relaxed: No ordering guarantees, _relaxed suffix.
 *
 * For compound atomics performing both a load and a store, ACQUIRE
 * semantics apply only to the load and RELEASE semantics only to the
 * store portion of the operation. Note that a failed cmpxchg_acquire
 * does -not- imply any memory ordering constraints.
 *
 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
 */

#ifndef atomic_read_acquire
#define  atomic_read_acquire(v)		smp_load_acquire(&(v)->counter)
#endif

#ifndef atomic_set_release
#define  atomic_set_release(v, i)	smp_store_release(&(v)->counter, (i))
#endif

/*
 * The idea here is to build acquire/release variants by adding explicit
 * barriers on top of the relaxed variant. In the case where the relaxed
 * variant is already fully ordered, no additional barriers are needed.
 *
 * Besides, if an arch has a special barrier for acquire/release, it could
 * implement its own __atomic_op_* and use the same framework for building
 * variants
 */
#ifndef __atomic_op_acquire
#define __atomic_op_acquire(op, args...)				\
({									\
	typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);		\
	smp_mb__after_atomic();						\
	__ret;								\
})
#endif

#ifndef __atomic_op_release
#define __atomic_op_release(op, args...)				\
({									\
	smp_mb__before_atomic();					\
	op##_relaxed(args);						\
})
#endif

#ifndef __atomic_op_fence
#define __atomic_op_fence(op, args...)					\
({									\
	typeof(op##_relaxed(args)) __ret;				\
	smp_mb__before_atomic();					\
	__ret = op##_relaxed(args);					\
	smp_mb__after_atomic();						\
	__ret;								\
})
#endif

/* atomic_add_return_relaxed */
#ifndef atomic_add_return_relaxed
#define  atomic_add_return_relaxed	atomic_add_return
#define  atomic_add_return_acquire	atomic_add_return
#define  atomic_add_return_release	atomic_add_return

#else /* atomic_add_return_relaxed */

#ifndef atomic_add_return_acquire
#define  atomic_add_return_acquire(...)					\
	__atomic_op_acquire(atomic_add_return, __VA_ARGS__)
#endif

#ifndef atomic_add_return_release
#define  atomic_add_return_release(...)					\
	__atomic_op_release(atomic_add_return, __VA_ARGS__)
#endif

#ifndef atomic_add_return
#define  atomic_add_return(...)						\
	__atomic_op_fence(atomic_add_return, __VA_ARGS__)
#endif
#endif /* atomic_add_return_relaxed */

/* atomic_inc_return_relaxed */
#ifndef atomic_inc_return_relaxed
#define  atomic_inc_return_relaxed	atomic_inc_return
#define  atomic_inc_return_acquire	atomic_inc_return
#define  atomic_inc_return_release	atomic_inc_return

#else /* atomic_inc_return_relaxed */

#ifndef atomic_inc_return_acquire
#define  atomic_inc_return_acquire(...)					\
	__atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
#endif

#ifndef atomic_inc_return_release
#define  atomic_inc_return_release(...)					\
	__atomic_op_release(atomic_inc_return, __VA_ARGS__)
#endif

#ifndef atomic_inc_return
#define  atomic_inc_return(...)						\
	__atomic_op_fence(atomic_inc_return, __VA_ARGS__)
#endif
#endif /* atomic_inc_return_relaxed */

/* atomic_sub_return_relaxed */
#ifndef atomic_sub_return_relaxed
#define  atomic_sub_return_relaxed	atomic_sub_return
#define  atomic_sub_return_acquire	atomic_sub_return
#define  atomic_sub_return_release	atomic_sub_return

#else /* atomic_sub_return_relaxed */

#ifndef atomic_sub_return_acquire
#define  atomic_sub_return_acquire(...)					\
	__atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
#endif

#ifndef atomic_sub_return_release
#define  atomic_sub_return_release(...)					\
	__atomic_op_release(atomic_sub_return, __VA_ARGS__)
#endif

#ifndef atomic_sub_return
#define  atomic_sub_return(...)						\
	__atomic_op_fence(atomic_sub_return, __VA_ARGS__)
#endif
#endif /* atomic_sub_return_relaxed */

/* atomic_dec_return_relaxed */
#ifndef atomic_dec_return_relaxed
#define  atomic_dec_return_relaxed	atomic_dec_return
#define  atomic_dec_return_acquire	atomic_dec_return
#define  atomic_dec_return_release	atomic_dec_return

#else /* atomic_dec_return_relaxed */

#ifndef atomic_dec_return_acquire
#define  atomic_dec_return_acquire(...)					\
	__atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
#endif

#ifndef atomic_dec_return_release
#define  atomic_dec_return_release(...)					\
	__atomic_op_release(atomic_dec_return, __VA_ARGS__)
#endif

#ifndef atomic_dec_return
#define  atomic_dec_return(...)						\
	__atomic_op_fence(atomic_dec_return, __VA_ARGS__)
#endif
#endif /* atomic_dec_return_relaxed */

/* atomic_xchg_relaxed */
#ifndef atomic_xchg_relaxed
#define  atomic_xchg_relaxed		atomic_xchg
#define  atomic_xchg_acquire		atomic_xchg
#define  atomic_xchg_release		atomic_xchg

#else /* atomic_xchg_relaxed */

#ifndef atomic_xchg_acquire
#define  atomic_xchg_acquire(...)					\
	__atomic_op_acquire(atomic_xchg, __VA_ARGS__)
#endif

#ifndef atomic_xchg_release
#define  atomic_xchg_release(...)					\
	__atomic_op_release(atomic_xchg, __VA_ARGS__)
#endif

#ifndef atomic_xchg
#define  atomic_xchg(...)						\
	__atomic_op_fence(atomic_xchg, __VA_ARGS__)
#endif
#endif /* atomic_xchg_relaxed */

/* atomic_cmpxchg_relaxed */
#ifndef atomic_cmpxchg_relaxed
#define  atomic_cmpxchg_relaxed		atomic_cmpxchg
#define  atomic_cmpxchg_acquire		atomic_cmpxchg
#define  atomic_cmpxchg_release		atomic_cmpxchg

#else /* atomic_cmpxchg_relaxed */

#ifndef atomic_cmpxchg_acquire
#define  atomic_cmpxchg_acquire(...)					\
	__atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
#endif

#ifndef atomic_cmpxchg_release
#define  atomic_cmpxchg_release(...)					\
	__atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
#endif

#ifndef atomic_cmpxchg
#define  atomic_cmpxchg(...)						\
	__atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
#endif
#endif /* atomic_cmpxchg_relaxed */

#ifndef atomic64_read_acquire
#define  atomic64_read_acquire(v)	smp_load_acquire(&(v)->counter)
#endif

#ifndef atomic64_set_release
#define  atomic64_set_release(v, i)	smp_store_release(&(v)->counter, (i))
#endif

/* atomic64_add_return_relaxed */
#ifndef atomic64_add_return_relaxed
#define  atomic64_add_return_relaxed	atomic64_add_return
#define  atomic64_add_return_acquire	atomic64_add_return
#define  atomic64_add_return_release	atomic64_add_return

#else /* atomic64_add_return_relaxed */

#ifndef atomic64_add_return_acquire
#define  atomic64_add_return_acquire(...)				\
	__atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
#endif

#ifndef atomic64_add_return_release
#define  atomic64_add_return_release(...)				\
	__atomic_op_release(atomic64_add_return, __VA_ARGS__)
#endif

#ifndef atomic64_add_return
#define  atomic64_add_return(...)					\
	__atomic_op_fence(atomic64_add_return, __VA_ARGS__)
#endif
#endif /* atomic64_add_return_relaxed */

/* atomic64_inc_return_relaxed */
#ifndef atomic64_inc_return_relaxed
#define  atomic64_inc_return_relaxed	atomic64_inc_return
#define  atomic64_inc_return_acquire	atomic64_inc_return
#define  atomic64_inc_return_release	atomic64_inc_return

#else /* atomic64_inc_return_relaxed */

#ifndef atomic64_inc_return_acquire
#define  atomic64_inc_return_acquire(...)				\
	__atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
#endif

#ifndef atomic64_inc_return_release
#define  atomic64_inc_return_release(...)				\
	__atomic_op_release(atomic64_inc_return, __VA_ARGS__)
#endif

#ifndef atomic64_inc_return
#define  atomic64_inc_return(...)					\
	__atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
#endif
#endif /* atomic64_inc_return_relaxed */


/* atomic64_sub_return_relaxed */
#ifndef atomic64_sub_return_relaxed
#define  atomic64_sub_return_relaxed	atomic64_sub_return
#define  atomic64_sub_return_acquire	atomic64_sub_return
#define  atomic64_sub_return_release	atomic64_sub_return

#else /* atomic64_sub_return_relaxed */

#ifndef atomic64_sub_return_acquire
#define  atomic64_sub_return_acquire(...)				\
	__atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
#endif

#ifndef atomic64_sub_return_release
#define  atomic64_sub_return_release(...)				\
	__atomic_op_release(atomic64_sub_return, __VA_ARGS__)
#endif

#ifndef atomic64_sub_return
#define  atomic64_sub_return(...)					\
	__atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
#endif
#endif /* atomic64_sub_return_relaxed */

/* atomic64_dec_return_relaxed */
#ifndef atomic64_dec_return_relaxed
#define  atomic64_dec_return_relaxed	atomic64_dec_return
#define  atomic64_dec_return_acquire	atomic64_dec_return
#define  atomic64_dec_return_release	atomic64_dec_return

#else /* atomic64_dec_return_relaxed */

#ifndef atomic64_dec_return_acquire
#define  atomic64_dec_return_acquire(...)				\
	__atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
#endif

#ifndef atomic64_dec_return_release
#define  atomic64_dec_return_release(...)				\
	__atomic_op_release(atomic64_dec_return, __VA_ARGS__)
#endif

#ifndef atomic64_dec_return
#define  atomic64_dec_return(...)					\
	__atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
#endif
#endif /* atomic64_dec_return_relaxed */

/* atomic64_xchg_relaxed */
#ifndef atomic64_xchg_relaxed
#define  atomic64_xchg_relaxed		atomic64_xchg
#define  atomic64_xchg_acquire		atomic64_xchg
#define  atomic64_xchg_release		atomic64_xchg

#else /* atomic64_xchg_relaxed */

#ifndef atomic64_xchg_acquire
#define  atomic64_xchg_acquire(...)					\
	__atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
#endif

#ifndef atomic64_xchg_release
#define  atomic64_xchg_release(...)					\
	__atomic_op_release(atomic64_xchg, __VA_ARGS__)
#endif

#ifndef atomic64_xchg
#define  atomic64_xchg(...)						\
	__atomic_op_fence(atomic64_xchg, __VA_ARGS__)
#endif
#endif /* atomic64_xchg_relaxed */

/* atomic64_cmpxchg_relaxed */
#ifndef atomic64_cmpxchg_relaxed
#define  atomic64_cmpxchg_relaxed	atomic64_cmpxchg
#define  atomic64_cmpxchg_acquire	atomic64_cmpxchg
#define  atomic64_cmpxchg_release	atomic64_cmpxchg

#else /* atomic64_cmpxchg_relaxed */

#ifndef atomic64_cmpxchg_acquire
#define  atomic64_cmpxchg_acquire(...)					\
	__atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
#endif

#ifndef atomic64_cmpxchg_release
#define  atomic64_cmpxchg_release(...)					\
	__atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
#endif

#ifndef atomic64_cmpxchg
#define  atomic64_cmpxchg(...)						\
	__atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
#endif
#endif /* atomic64_cmpxchg_relaxed */

/* cmpxchg_relaxed */
#ifndef cmpxchg_relaxed
#define  cmpxchg_relaxed		cmpxchg
#define  cmpxchg_acquire		cmpxchg
#define  cmpxchg_release		cmpxchg

#else /* cmpxchg_relaxed */

#ifndef cmpxchg_acquire
#define  cmpxchg_acquire(...)						\
	__atomic_op_acquire(cmpxchg, __VA_ARGS__)
#endif

#ifndef cmpxchg_release
#define  cmpxchg_release(...)						\
	__atomic_op_release(cmpxchg, __VA_ARGS__)
#endif

#ifndef cmpxchg
#define  cmpxchg(...)							\
	__atomic_op_fence(cmpxchg, __VA_ARGS__)
#endif
#endif /* cmpxchg_relaxed */

/* cmpxchg64_relaxed */
#ifndef cmpxchg64_relaxed
#define  cmpxchg64_relaxed		cmpxchg64
#define  cmpxchg64_acquire		cmpxchg64
#define  cmpxchg64_release		cmpxchg64

#else /* cmpxchg64_relaxed */

#ifndef cmpxchg64_acquire
#define  cmpxchg64_acquire(...)						\
	__atomic_op_acquire(cmpxchg64, __VA_ARGS__)
#endif

#ifndef cmpxchg64_release
#define  cmpxchg64_release(...)						\
	__atomic_op_release(cmpxchg64, __VA_ARGS__)
#endif

#ifndef cmpxchg64
#define  cmpxchg64(...)							\
	__atomic_op_fence(cmpxchg64, __VA_ARGS__)
#endif
#endif /* cmpxchg64_relaxed */

/* xchg_relaxed */
#ifndef xchg_relaxed
#define  xchg_relaxed			xchg
#define  xchg_acquire			xchg
#define  xchg_release			xchg

#else /* xchg_relaxed */

#ifndef xchg_acquire
#define  xchg_acquire(...)		__atomic_op_acquire(xchg, __VA_ARGS__)
#endif

#ifndef xchg_release
#define  xchg_release(...)		__atomic_op_release(xchg, __VA_ARGS__)
#endif

#ifndef xchg
#define  xchg(...)			__atomic_op_fence(xchg, __VA_ARGS__)
#endif
#endif /* xchg_relaxed */

/**
 * atomic_add_unless - add unless the number is already a given value
 * @v: pointer of type atomic_t
 * @a: the amount to add to v...
 * @u: ...unless v is equal to u.
 *
 * Atomically adds @a to @v, so long as @v was not already @u.
 * Returns non-zero if @v was not @u, and zero otherwise.
 */
static inline int atomic_add_unless(atomic_t *v, int a, int u)
{
	return __atomic_add_unless(v, a, u) != u;
}

/**
 * atomic_inc_not_zero - increment unless the number is zero
 * @v: pointer of type atomic_t
 *
 * Atomically increments @v by 1, so long as @v is non-zero.
 * Returns non-zero if @v was non-zero, and zero otherwise.
 */
#ifndef atomic_inc_not_zero
#define atomic_inc_not_zero(v)		atomic_add_unless((v), 1, 0)
#endif

#ifndef atomic_andnot
static inline void atomic_andnot(int i, atomic_t *v)
{
	atomic_and(~i, v);
}
#endif

static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
{
	atomic_andnot(mask, v);
}

static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v)
{
	atomic_or(mask, v);
}

/**
 * atomic_inc_not_zero_hint - increment if not null
 * @v: pointer of type atomic_t
 * @hint: probable value of the atomic before the increment
 *
 * This version of atomic_inc_not_zero() gives a hint of probable
 * value of the atomic. This helps processor to not read the memory
 * before doing the atomic read/modify/write cycle, lowering
 * number of bus transactions on some arches.
 *
 * Returns: 0 if increment was not done, 1 otherwise.
 */
#ifndef atomic_inc_not_zero_hint
static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
{
	int val, c = hint;

	/* sanity test, should be removed by compiler if hint is a constant */
	if (!hint)
		return atomic_inc_not_zero(v);

	do {
		val = atomic_cmpxchg(v, c, c + 1);
		if (val == c)
			return 1;
		c = val;
	} while (c);

	return 0;
}
#endif

#ifndef atomic_inc_unless_negative
static inline int atomic_inc_unless_negative(atomic_t *p)
{
	int v, v1;
	for (v = 0; v >= 0; v = v1) {
		v1 = atomic_cmpxchg(p, v, v + 1);
		if (likely(v1 == v))
			return 1;
	}
	return 0;
}
#endif

#ifndef atomic_dec_unless_positive
static inline int atomic_dec_unless_positive(atomic_t *p)
{
	int v, v1;
	for (v = 0; v <= 0; v = v1) {
		v1 = atomic_cmpxchg(p, v, v - 1);
		if (likely(v1 == v))
			return 1;
	}
	return 0;
}
#endif

/*
 * atomic_dec_if_positive - decrement by 1 if old value positive
 * @v: pointer of type atomic_t
 *
 * The function returns the old value of *v minus 1, even if
 * the atomic variable, v, was not decremented.
 */
#ifndef atomic_dec_if_positive
static inline int atomic_dec_if_positive(atomic_t *v)
{
	int c, old, dec;
	c = atomic_read(v);
	for (;;) {
		dec = c - 1;
		if (unlikely(dec < 0))
			break;
		old = atomic_cmpxchg((v), c, dec);
		if (likely(old == c))
			break;
		c = old;
	}
	return dec;
}
#endif

/**
 * fetch_or - perform *ptr |= mask and return old value of *ptr
 * @ptr: pointer to value
 * @mask: mask to OR on the value
 *
 * cmpxchg based fetch_or, macro so it works for different integer types
 */
#ifndef fetch_or
#define fetch_or(ptr, mask)						\
({	typeof(*(ptr)) __old, __val = *(ptr);				\
	for (;;) {							\
		__old = cmpxchg((ptr), __val, __val | (mask));		\
		if (__old == __val)					\
			break;						\
		__val = __old;						\
	}								\
	__old;								\
})
#endif


#ifdef CONFIG_GENERIC_ATOMIC64
#include <asm-generic/atomic64.h>
#endif

#ifndef atomic64_andnot
static inline void atomic64_andnot(long long i, atomic64_t *v)
{
	atomic64_and(~i, v);
}
#endif

#include <asm-generic/atomic-long.h>

#endif /* _LINUX_ATOMIC_H */
Commit	Line	Data
acac43e2	1	/* Atomic operations usable in machine independent code */
3f9d35b9 ED	2	#ifndef _LINUX_ATOMIC_H
	3	#define _LINUX_ATOMIC_H
	4	#include <asm/atomic.h>
654672d4 WD	5	#include <asm/barrier.h>
	6
	7	/*
	8	* Relaxed variants of xchg, cmpxchg and some atomic operations.
	9	*
	10	* We support four variants:
	11	*
	12	* - Fully ordered: The default implementation, no suffix required.
	13	* - Acquire: Provides ACQUIRE semantics, _acquire suffix.
	14	* - Release: Provides RELEASE semantics, _release suffix.
	15	* - Relaxed: No ordering guarantees, _relaxed suffix.
	16	*
	17	* For compound atomics performing both a load and a store, ACQUIRE
	18	* semantics apply only to the load and RELEASE semantics only to the
	19	* store portion of the operation. Note that a failed cmpxchg_acquire
	20	* does -not- imply any memory ordering constraints.
	21	*
	22	* See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
	23	*/
	24
	25	#ifndef atomic_read_acquire
	26	#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
	27	#endif
	28
	29	#ifndef atomic_set_release
	30	#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
	31	#endif
	32
	33	/*
	34	* The idea here is to build acquire/release variants by adding explicit
	35	* barriers on top of the relaxed variant. In the case where the relaxed
	36	* variant is already fully ordered, no additional barriers are needed.
e1ab7f39 BF	37	*
	38	* Besides, if an arch has a special barrier for acquire/release, it could
	39	* implement its own __atomic_op_* and use the same framework for building
	40	* variants
654672d4	41	*/
e1ab7f39	42	#ifndef __atomic_op_acquire
654672d4 WD	43	#define __atomic_op_acquire(op, args...) \
	44	({ \
	45	typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
	46	smp_mb__after_atomic(); \
	47	__ret; \
	48	})
e1ab7f39	49	#endif
654672d4	50
e1ab7f39	51	#ifndef __atomic_op_release
654672d4 WD	52	#define __atomic_op_release(op, args...) \
	53	({ \
	54	smp_mb__before_atomic(); \
	55	op##_relaxed(args); \
	56	})
e1ab7f39	57	#endif
654672d4	58
e1ab7f39	59	#ifndef __atomic_op_fence
654672d4 WD	60	#define __atomic_op_fence(op, args...) \
	61	({ \
	62	typeof(op##_relaxed(args)) __ret; \
	63	smp_mb__before_atomic(); \
	64	__ret = op##_relaxed(args); \
	65	smp_mb__after_atomic(); \
	66	__ret; \
	67	})
e1ab7f39	68	#endif
654672d4 WD	69
	70	/* atomic_add_return_relaxed */
	71	#ifndef atomic_add_return_relaxed
	72	#define atomic_add_return_relaxed atomic_add_return
	73	#define atomic_add_return_acquire atomic_add_return
	74	#define atomic_add_return_release atomic_add_return
	75
	76	#else /* atomic_add_return_relaxed */
	77
	78	#ifndef atomic_add_return_acquire
	79	#define atomic_add_return_acquire(...) \
	80	__atomic_op_acquire(atomic_add_return, __VA_ARGS__)
	81	#endif
	82
	83	#ifndef atomic_add_return_release
	84	#define atomic_add_return_release(...) \
	85	__atomic_op_release(atomic_add_return, __VA_ARGS__)
	86	#endif
	87
	88	#ifndef atomic_add_return
	89	#define atomic_add_return(...) \
	90	__atomic_op_fence(atomic_add_return, __VA_ARGS__)
	91	#endif
	92	#endif /* atomic_add_return_relaxed */
	93
63ab7bd0 DB	94	/* atomic_inc_return_relaxed */
	95	#ifndef atomic_inc_return_relaxed
	96	#define atomic_inc_return_relaxed atomic_inc_return
	97	#define atomic_inc_return_acquire atomic_inc_return
	98	#define atomic_inc_return_release atomic_inc_return
	99
	100	#else /* atomic_inc_return_relaxed */
	101
	102	#ifndef atomic_inc_return_acquire
	103	#define atomic_inc_return_acquire(...) \
	104	__atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
	105	#endif
	106
	107	#ifndef atomic_inc_return_release
	108	#define atomic_inc_return_release(...) \
	109	__atomic_op_release(atomic_inc_return, __VA_ARGS__)
	110	#endif
	111
	112	#ifndef atomic_inc_return
	113	#define atomic_inc_return(...) \
	114	__atomic_op_fence(atomic_inc_return, __VA_ARGS__)
	115	#endif
	116	#endif /* atomic_inc_return_relaxed */
	117
654672d4 WD	118	/* atomic_sub_return_relaxed */
	119	#ifndef atomic_sub_return_relaxed
	120	#define atomic_sub_return_relaxed atomic_sub_return
	121	#define atomic_sub_return_acquire atomic_sub_return
	122	#define atomic_sub_return_release atomic_sub_return
	123
	124	#else /* atomic_sub_return_relaxed */
	125
	126	#ifndef atomic_sub_return_acquire
	127	#define atomic_sub_return_acquire(...) \
	128	__atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
	129	#endif
	130
	131	#ifndef atomic_sub_return_release
	132	#define atomic_sub_return_release(...) \
	133	__atomic_op_release(atomic_sub_return, __VA_ARGS__)
	134	#endif
	135
	136	#ifndef atomic_sub_return
	137	#define atomic_sub_return(...) \
	138	__atomic_op_fence(atomic_sub_return, __VA_ARGS__)
	139	#endif
	140	#endif /* atomic_sub_return_relaxed */
	141
63ab7bd0 DB	142	/* atomic_dec_return_relaxed */
	143	#ifndef atomic_dec_return_relaxed
	144	#define atomic_dec_return_relaxed atomic_dec_return
	145	#define atomic_dec_return_acquire atomic_dec_return
	146	#define atomic_dec_return_release atomic_dec_return
	147
	148	#else /* atomic_dec_return_relaxed */
	149
	150	#ifndef atomic_dec_return_acquire
	151	#define atomic_dec_return_acquire(...) \
	152	__atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
	153	#endif
	154
	155	#ifndef atomic_dec_return_release
	156	#define atomic_dec_return_release(...) \
	157	__atomic_op_release(atomic_dec_return, __VA_ARGS__)
	158	#endif
	159
	160	#ifndef atomic_dec_return
	161	#define atomic_dec_return(...) \
	162	__atomic_op_fence(atomic_dec_return, __VA_ARGS__)
	163	#endif
	164	#endif /* atomic_dec_return_relaxed */
	165
654672d4 WD	166	/* atomic_xchg_relaxed */
	167	#ifndef atomic_xchg_relaxed
	168	#define atomic_xchg_relaxed atomic_xchg
	169	#define atomic_xchg_acquire atomic_xchg
	170	#define atomic_xchg_release atomic_xchg
	171
	172	#else /* atomic_xchg_relaxed */
	173
	174	#ifndef atomic_xchg_acquire
	175	#define atomic_xchg_acquire(...) \
	176	__atomic_op_acquire(atomic_xchg, __VA_ARGS__)
	177	#endif
	178
	179	#ifndef atomic_xchg_release
	180	#define atomic_xchg_release(...) \
	181	__atomic_op_release(atomic_xchg, __VA_ARGS__)
	182	#endif
	183
	184	#ifndef atomic_xchg
	185	#define atomic_xchg(...) \
	186	__atomic_op_fence(atomic_xchg, __VA_ARGS__)
	187	#endif
	188	#endif /* atomic_xchg_relaxed */
	189
	190	/* atomic_cmpxchg_relaxed */
	191	#ifndef atomic_cmpxchg_relaxed
	192	#define atomic_cmpxchg_relaxed atomic_cmpxchg
	193	#define atomic_cmpxchg_acquire atomic_cmpxchg
	194	#define atomic_cmpxchg_release atomic_cmpxchg
	195
	196	#else /* atomic_cmpxchg_relaxed */
	197
	198	#ifndef atomic_cmpxchg_acquire
	199	#define atomic_cmpxchg_acquire(...) \
	200	__atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
	201	#endif
	202
	203	#ifndef atomic_cmpxchg_release
	204	#define atomic_cmpxchg_release(...) \
	205	__atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
	206	#endif
	207
	208	#ifndef atomic_cmpxchg
	209	#define atomic_cmpxchg(...) \
	210	__atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
	211	#endif
	212	#endif /* atomic_cmpxchg_relaxed */
	213
	214	#ifndef atomic64_read_acquire
	215	#define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter)
	216	#endif
	217
	218	#ifndef atomic64_set_release
	219	#define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i))
	220	#endif
	221
	222	/* atomic64_add_return_relaxed */
	223	#ifndef atomic64_add_return_relaxed
	224	#define atomic64_add_return_relaxed atomic64_add_return
	225	#define atomic64_add_return_acquire atomic64_add_return
	226	#define atomic64_add_return_release atomic64_add_return
	227
	228	#else /* atomic64_add_return_relaxed */
	229
230	#ifndef atomic64_add_return_acquire
231	#define atomic64_add_return_acquire(...) \
232	__atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
233	#endif
234
235	#ifndef atomic64_add_return_release
236	#define atomic64_add_return_release(...) \
237	__atomic_op_release(atomic64_add_return, __VA_ARGS__)
238	#endif
239
240	#ifndef atomic64_add_return
241	#define atomic64_add_return(...) \
242	__atomic_op_fence(atomic64_add_return, __VA_ARGS__)
243	#endif
244	#endif /* atomic64_add_return_relaxed */
245
63ab7bd0 DB	246	/* atomic64_inc_return_relaxed */
	247	#ifndef atomic64_inc_return_relaxed
	248	#define atomic64_inc_return_relaxed atomic64_inc_return
	249	#define atomic64_inc_return_acquire atomic64_inc_return
	250	#define atomic64_inc_return_release atomic64_inc_return
	251
	252	#else /* atomic64_inc_return_relaxed */
	253
	254	#ifndef atomic64_inc_return_acquire
	255	#define atomic64_inc_return_acquire(...) \
	256	__atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
	257	#endif
	258
	259	#ifndef atomic64_inc_return_release
	260	#define atomic64_inc_return_release(...) \
	261	__atomic_op_release(atomic64_inc_return, __VA_ARGS__)
	262	#endif
	263
	264	#ifndef atomic64_inc_return
	265	#define atomic64_inc_return(...) \
	266	__atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
	267	#endif
	268	#endif /* atomic64_inc_return_relaxed */
	269
	270
654672d4 WD	271	/* atomic64_sub_return_relaxed */
	272	#ifndef atomic64_sub_return_relaxed
	273	#define atomic64_sub_return_relaxed atomic64_sub_return
	274	#define atomic64_sub_return_acquire atomic64_sub_return
	275	#define atomic64_sub_return_release atomic64_sub_return
	276
	277	#else /* atomic64_sub_return_relaxed */
	278
	279	#ifndef atomic64_sub_return_acquire
	280	#define atomic64_sub_return_acquire(...) \
	281	__atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
	282	#endif
	283
	284	#ifndef atomic64_sub_return_release
	285	#define atomic64_sub_return_release(...) \
	286	__atomic_op_release(atomic64_sub_return, __VA_ARGS__)
	287	#endif
	288
	289	#ifndef atomic64_sub_return
	290	#define atomic64_sub_return(...) \
	291	__atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
	292	#endif
	293	#endif /* atomic64_sub_return_relaxed */
	294
63ab7bd0 DB	295	/* atomic64_dec_return_relaxed */
	296	#ifndef atomic64_dec_return_relaxed
	297	#define atomic64_dec_return_relaxed atomic64_dec_return
	298	#define atomic64_dec_return_acquire atomic64_dec_return
	299	#define atomic64_dec_return_release atomic64_dec_return
	300
	301	#else /* atomic64_dec_return_relaxed */
	302
	303	#ifndef atomic64_dec_return_acquire
	304	#define atomic64_dec_return_acquire(...) \
	305	__atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
	306	#endif
	307
	308	#ifndef atomic64_dec_return_release
	309	#define atomic64_dec_return_release(...) \
	310	__atomic_op_release(atomic64_dec_return, __VA_ARGS__)
	311	#endif
	312
	313	#ifndef atomic64_dec_return
	314	#define atomic64_dec_return(...) \
	315	__atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
	316	#endif
	317	#endif /* atomic64_dec_return_relaxed */
	318
654672d4 WD	319	/* atomic64_xchg_relaxed */
	320	#ifndef atomic64_xchg_relaxed
	321	#define atomic64_xchg_relaxed atomic64_xchg
	322	#define atomic64_xchg_acquire atomic64_xchg
	323	#define atomic64_xchg_release atomic64_xchg
	324
	325	#else /* atomic64_xchg_relaxed */
	326
	327	#ifndef atomic64_xchg_acquire
	328	#define atomic64_xchg_acquire(...) \
	329	__atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
	330	#endif
	331
	332	#ifndef atomic64_xchg_release
	333	#define atomic64_xchg_release(...) \
	334	__atomic_op_release(atomic64_xchg, __VA_ARGS__)
	335	#endif
	336
	337	#ifndef atomic64_xchg
	338	#define atomic64_xchg(...) \
	339	__atomic_op_fence(atomic64_xchg, __VA_ARGS__)
	340	#endif
	341	#endif /* atomic64_xchg_relaxed */
	342
	343	/* atomic64_cmpxchg_relaxed */
	344	#ifndef atomic64_cmpxchg_relaxed
	345	#define atomic64_cmpxchg_relaxed atomic64_cmpxchg
	346	#define atomic64_cmpxchg_acquire atomic64_cmpxchg
	347	#define atomic64_cmpxchg_release atomic64_cmpxchg
	348
	349	#else /* atomic64_cmpxchg_relaxed */
	350
	351	#ifndef atomic64_cmpxchg_acquire
	352	#define atomic64_cmpxchg_acquire(...) \
	353	__atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
	354	#endif
	355
	356	#ifndef atomic64_cmpxchg_release
	357	#define atomic64_cmpxchg_release(...) \
	358	__atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
	359	#endif
	360
	361	#ifndef atomic64_cmpxchg
	362	#define atomic64_cmpxchg(...) \
	363	__atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
	364	#endif
	365	#endif /* atomic64_cmpxchg_relaxed */
	366
	367	/* cmpxchg_relaxed */
	368	#ifndef cmpxchg_relaxed
	369	#define cmpxchg_relaxed cmpxchg
	370	#define cmpxchg_acquire cmpxchg
	371	#define cmpxchg_release cmpxchg
	372
	373	#else /* cmpxchg_relaxed */
	374
	375	#ifndef cmpxchg_acquire
	376	#define cmpxchg_acquire(...) \
	377	__atomic_op_acquire(cmpxchg, __VA_ARGS__)
	378	#endif
	379
	380	#ifndef cmpxchg_release
	381	#define cmpxchg_release(...) \
	382	__atomic_op_release(cmpxchg, __VA_ARGS__)
383	#endif
384
385	#ifndef cmpxchg
386	#define cmpxchg(...) \
387	__atomic_op_fence(cmpxchg, __VA_ARGS__)
388	#endif
389	#endif /* cmpxchg_relaxed */
390
391	/* cmpxchg64_relaxed */
392	#ifndef cmpxchg64_relaxed
393	#define cmpxchg64_relaxed cmpxchg64
394	#define cmpxchg64_acquire cmpxchg64
395	#define cmpxchg64_release cmpxchg64
396
397	#else /* cmpxchg64_relaxed */
398
399	#ifndef cmpxchg64_acquire
400	#define cmpxchg64_acquire(...) \
401	__atomic_op_acquire(cmpxchg64, __VA_ARGS__)
402	#endif
403
404	#ifndef cmpxchg64_release
405	#define cmpxchg64_release(...) \
406	__atomic_op_release(cmpxchg64, __VA_ARGS__)
407	#endif
408
409	#ifndef cmpxchg64
410	#define cmpxchg64(...) \
411	__atomic_op_fence(cmpxchg64, __VA_ARGS__)
412	#endif
413	#endif /* cmpxchg64_relaxed */
414
415	/* xchg_relaxed */
416	#ifndef xchg_relaxed
417	#define xchg_relaxed xchg
418	#define xchg_acquire xchg
419	#define xchg_release xchg
420
421	#else /* xchg_relaxed */
422
423	#ifndef xchg_acquire
424	#define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__)
425	#endif
426
427	#ifndef xchg_release
428	#define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__)
429	#endif
430
431	#ifndef xchg
432	#define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__)
433	#endif
434	#endif /* xchg_relaxed */
3f9d35b9	435
f24219b4 AS	436	/**
	437	* atomic_add_unless - add unless the number is already a given value
	438	* @v: pointer of type atomic_t
	439	* @a: the amount to add to v...
	440	* @u: ...unless v is equal to u.
	441	*
	442	* Atomically adds @a to @v, so long as @v was not already @u.
	443	* Returns non-zero if @v was not @u, and zero otherwise.
	444	*/
	445	static inline int atomic_add_unless(atomic_t *v, int a, int u)
	446	{
	447	return __atomic_add_unless(v, a, u) != u;
	448	}
	449
60063497 AS	450	/**
	451	* atomic_inc_not_zero - increment unless the number is zero
	452	* @v: pointer of type atomic_t
	453	*
	454	* Atomically increments @v by 1, so long as @v is non-zero.
	455	* Returns non-zero if @v was non-zero, and zero otherwise.
	456	*/
b1ada601	457	#ifndef atomic_inc_not_zero
60063497	458	#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
b1ada601	459	#endif
60063497	460
de9e432c PZ	461	#ifndef atomic_andnot
	462	static inline void atomic_andnot(int i, atomic_t *v)
	463	{
	464	atomic_and(~i, v);
	465	}
	466	#endif
	467
	468	static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
	469	{
	470	atomic_andnot(mask, v);
	471	}
	472
	473	static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v)
	474	{
	475	atomic_or(mask, v);
	476	}
	477
3f9d35b9 ED	478	/**
	479	* atomic_inc_not_zero_hint - increment if not null
	480	* @v: pointer of type atomic_t
	481	* @hint: probable value of the atomic before the increment
	482	*
	483	* This version of atomic_inc_not_zero() gives a hint of probable
	484	* value of the atomic. This helps processor to not read the memory
	485	* before doing the atomic read/modify/write cycle, lowering
	486	* number of bus transactions on some arches.
	487	*
	488	* Returns: 0 if increment was not done, 1 otherwise.
	489	*/
	490	#ifndef atomic_inc_not_zero_hint
	491	static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
	492	{
	493	int val, c = hint;
	494
	495	/* sanity test, should be removed by compiler if hint is a constant */
	496	if (!hint)
	497	return atomic_inc_not_zero(v);
	498
	499	do {
	500	val = atomic_cmpxchg(v, c, c + 1);
	501	if (val == c)
	502	return 1;
	503	c = val;
	504	} while (c);
	505
	506	return 0;
	507	}
	508	#endif
	509
07b8ce1e AV	510	#ifndef atomic_inc_unless_negative
	511	static inline int atomic_inc_unless_negative(atomic_t *p)
	512	{
	513	int v, v1;
	514	for (v = 0; v >= 0; v = v1) {
	515	v1 = atomic_cmpxchg(p, v, v + 1);
	516	if (likely(v1 == v))
	517	return 1;
	518	}
	519	return 0;
	520	}
	521	#endif
	522
	523	#ifndef atomic_dec_unless_positive
	524	static inline int atomic_dec_unless_positive(atomic_t *p)
	525	{
	526	int v, v1;
	527	for (v = 0; v <= 0; v = v1) {
	528	v1 = atomic_cmpxchg(p, v, v - 1);
	529	if (likely(v1 == v))
	530	return 1;
	531	}
	532	return 0;
	533	}
	534	#endif
	535
e79bee24 SL	536	/*
	537	* atomic_dec_if_positive - decrement by 1 if old value positive
	538	* @v: pointer of type atomic_t
	539	*
	540	* The function returns the old value of *v minus 1, even if
	541	* the atomic variable, v, was not decremented.
	542	*/
	543	#ifndef atomic_dec_if_positive
	544	static inline int atomic_dec_if_positive(atomic_t *v)
	545	{
	546	int c, old, dec;
	547	c = atomic_read(v);
	548	for (;;) {
	549	dec = c - 1;
	550	if (unlikely(dec < 0))
	551	break;
	552	old = atomic_cmpxchg((v), c, dec);
	553	if (likely(old == c))
	554	break;
	555	c = old;
	556	}
	557	return dec;
	558	}
	559	#endif
	560
5fd7a09c FW	561	/**
	562	* fetch_or - perform ptr \|= mask and return old value of ptr
	563	* @ptr: pointer to value
	564	* @mask: mask to OR on the value
	565	*
	566	* cmpxchg based fetch_or, macro so it works for different integer types
	567	*/
	568	#ifndef fetch_or
	569	#define fetch_or(ptr, mask) \
	570	({ typeof((ptr)) __old, __val = (ptr); \
	571	for (;;) { \
	572	__old = cmpxchg((ptr), __val, __val \| (mask)); \
	573	if (__old == __val) \
	574	break; \
	575	__val = __old; \
	576	} \
	577	__old; \
	578	})
	579	#endif
	580
	581
7847777a AS	582	#ifdef CONFIG_GENERIC_ATOMIC64
	583	#include <asm-generic/atomic64.h>
	584	#endif
de9e432c PZ	585
	586	#ifndef atomic64_andnot
	587	static inline void atomic64_andnot(long long i, atomic64_t *v)
	588	{
	589	atomic64_and(~i, v);
	590	}
	591	#endif
	592
90fe6514 PZ	593	#include <asm-generic/atomic-long.h>
90fe6514 PZ	594
3f9d35b9	595	#endif /* _LINUX_ATOMIC_H */