[linux-2.6-block.git] / include / asm-s390 / atomic.h

#ifndef __ARCH_S390_ATOMIC__
#define __ARCH_S390_ATOMIC__

/*
 *  include/asm-s390/atomic.h
 *
 *  S390 version
 *    Copyright (C) 1999-2005 IBM Deutschland Entwicklung GmbH, IBM Corporation
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
 *               Denis Joseph Barrow,
 *		 Arnd Bergmann (arndb@de.ibm.com)
 *
 *  Derived from "include/asm-i386/bitops.h"
 *    Copyright (C) 1992, Linus Torvalds
 *
 */

/*
 * Atomic operations that C can't guarantee us.  Useful for
 * resource counting etc..
 * S390 uses 'Compare And Swap' for atomicity in SMP enviroment
 */

typedef struct {
	volatile int counter;
} __attribute__ ((aligned (4))) atomic_t;
#define ATOMIC_INIT(i)  { (i) }

#ifdef __KERNEL__

#define __CS_LOOP(ptr, op_val, op_string) ({				\
	typeof(ptr->counter) old_val, new_val;				\
        __asm__ __volatile__("   l     %0,0(%3)\n"			\
                             "0: lr    %1,%0\n"				\
                             op_string "  %1,%4\n"			\
                             "   cs    %0,%1,0(%3)\n"			\
                             "   jl    0b"				\
                             : "=&d" (old_val), "=&d" (new_val),	\
			       "=m" (((atomic_t *)(ptr))->counter)	\
			     : "a" (ptr), "d" (op_val),			\
			       "m" (((atomic_t *)(ptr))->counter)	\
			     : "cc", "memory" );			\
	new_val;							\
})
#define atomic_read(v)          ((v)->counter)
#define atomic_set(v,i)         (((v)->counter) = (i))

static __inline__ int atomic_add_return(int i, atomic_t * v)
{
	return __CS_LOOP(v, i, "ar");
}
#define atomic_add(_i, _v)		atomic_add_return(_i, _v)
#define atomic_add_negative(_i, _v)	(atomic_add_return(_i, _v) < 0)
#define atomic_inc(_v)			atomic_add_return(1, _v)
#define atomic_inc_return(_v)		atomic_add_return(1, _v)
#define atomic_inc_and_test(_v)		(atomic_add_return(1, _v) == 0)

static __inline__ int atomic_sub_return(int i, atomic_t * v)
{
	return __CS_LOOP(v, i, "sr");
}
#define atomic_sub(_i, _v)		atomic_sub_return(_i, _v)
#define atomic_sub_and_test(_i, _v)	(atomic_sub_return(_i, _v) == 0)
#define atomic_dec(_v)			atomic_sub_return(1, _v)
#define atomic_dec_return(_v)		atomic_sub_return(1, _v)
#define atomic_dec_and_test(_v)		(atomic_sub_return(1, _v) == 0)

static __inline__ void atomic_clear_mask(unsigned long mask, atomic_t * v)
{
	       __CS_LOOP(v, ~mask, "nr");
}

static __inline__ void atomic_set_mask(unsigned long mask, atomic_t * v)
{
	       __CS_LOOP(v, mask, "or");
}

#define atomic_xchg(v, new) (xchg(&((v)->counter), new))

static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new)
{
	__asm__ __volatile__("  cs   %0,%3,0(%2)\n"
			     : "+d" (old), "=m" (v->counter)
			     : "a" (v), "d" (new), "m" (v->counter)
			     : "cc", "memory" );
	return old;
}

static __inline__ int atomic_add_unless(atomic_t *v, int a, int u)
{
	int c, old;
	c = atomic_read(v);
	for (;;) {
		if (unlikely(c == u))
			break;
		old = atomic_cmpxchg(v, c, c + a);
		if (likely(old == c))
			break;
		c = old;
	}
	return c != u;
}

#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)

#undef __CS_LOOP

#ifdef __s390x__
typedef struct {
	volatile long long counter;
} __attribute__ ((aligned (8))) atomic64_t;
#define ATOMIC64_INIT(i)  { (i) }

#define __CSG_LOOP(ptr, op_val, op_string) ({				\
	typeof(ptr->counter) old_val, new_val;				\
        __asm__ __volatile__("   lg    %0,0(%3)\n"			\
                             "0: lgr   %1,%0\n"				\
                             op_string "  %1,%4\n"			\
                             "   csg   %0,%1,0(%3)\n"			\
                             "   jl    0b"				\
                             : "=&d" (old_val), "=&d" (new_val),	\
			       "=m" (((atomic_t *)(ptr))->counter)	\
			     : "a" (ptr), "d" (op_val),			\
			       "m" (((atomic_t *)(ptr))->counter)	\
			     : "cc", "memory" );			\
	new_val;							\
})
#define atomic64_read(v)          ((v)->counter)
#define atomic64_set(v,i)         (((v)->counter) = (i))

static __inline__ long long atomic64_add_return(long long i, atomic64_t * v)
{
	return __CSG_LOOP(v, i, "agr");
}
#define atomic64_add(_i, _v)		atomic64_add_return(_i, _v)
#define atomic64_add_negative(_i, _v)	(atomic64_add_return(_i, _v) < 0)
#define atomic64_inc(_v)		atomic64_add_return(1, _v)
#define atomic64_inc_return(_v)		atomic64_add_return(1, _v)
#define atomic64_inc_and_test(_v)	(atomic64_add_return(1, _v) == 0)

static __inline__ long long atomic64_sub_return(long long i, atomic64_t * v)
{
	return __CSG_LOOP(v, i, "sgr");
}
#define atomic64_sub(_i, _v)		atomic64_sub_return(_i, _v)
#define atomic64_sub_and_test(_i, _v)	(atomic64_sub_return(_i, _v) == 0)
#define atomic64_dec(_v)		atomic64_sub_return(1, _v)
#define atomic64_dec_return(_v)		atomic64_sub_return(1, _v)
#define atomic64_dec_and_test(_v)	(atomic64_sub_return(1, _v) == 0)

static __inline__ void atomic64_clear_mask(unsigned long mask, atomic64_t * v)
{
	       __CSG_LOOP(v, ~mask, "ngr");
}

static __inline__ void atomic64_set_mask(unsigned long mask, atomic64_t * v)
{
	       __CSG_LOOP(v, mask, "ogr");
}

static __inline__ long long atomic64_cmpxchg(atomic64_t *v,
					     long long old, long long new)
{
	__asm__ __volatile__("  csg  %0,%3,0(%2)\n"
			     : "+d" (old), "=m" (v->counter)
			     : "a" (v), "d" (new), "m" (v->counter)
			     : "cc", "memory" );
	return old;
}

static __inline__ int atomic64_add_unless(atomic64_t *v,
					  long long a, long long u)
{
	long long c, old;
	c = atomic64_read(v);
	for (;;) {
		if (unlikely(c == u))
			break;
		old = atomic64_cmpxchg(v, c, c + a);
		if (likely(old == c))
			break;
		c = old;
	}
	return c != u;
}

#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)

#undef __CSG_LOOP
#endif

#define smp_mb__before_atomic_dec()	smp_mb()
#define smp_mb__after_atomic_dec()	smp_mb()
#define smp_mb__before_atomic_inc()	smp_mb()
#define smp_mb__after_atomic_inc()	smp_mb()

#include <asm-generic/atomic.h>
#endif /* __KERNEL__ */
#endif /* __ARCH_S390_ATOMIC__  */
Commit	Line	Data
1da177e4 LT	1	#ifndef __ARCH_S390_ATOMIC__
	2	#define __ARCH_S390_ATOMIC__
	3
	4	/*
	5	* include/asm-s390/atomic.h
	6	*
	7	* S390 version
973bd993	8	* Copyright (C) 1999-2005 IBM Deutschland Entwicklung GmbH, IBM Corporation
1da177e4 LT	9	* Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
	10	* Denis Joseph Barrow,
	11	* Arnd Bergmann (arndb@de.ibm.com)
	12	*
	13	* Derived from "include/asm-i386/bitops.h"
	14	* Copyright (C) 1992, Linus Torvalds
	15	*
	16	*/
	17
	18	/*
	19	* Atomic operations that C can't guarantee us. Useful for
	20	* resource counting etc..
	21	* S390 uses 'Compare And Swap' for atomicity in SMP enviroment
	22	*/
	23
	24	typedef struct {
	25	volatile int counter;
	26	} __attribute__ ((aligned (4))) atomic_t;
	27	#define ATOMIC_INIT(i) { (i) }
	28
	29	#ifdef __KERNEL__
	30
	31	#define __CS_LOOP(ptr, op_val, op_string) ({ \
	32	typeof(ptr->counter) old_val, new_val; \
	33	__asm__ __volatile__(" l %0,0(%3)\n" \
	34	"0: lr %1,%0\n" \
	35	op_string " %1,%4\n" \
	36	" cs %0,%1,0(%3)\n" \
	37	" jl 0b" \
	38	: "=&d" (old_val), "=&d" (new_val), \
	39	"=m" (((atomic_t *)(ptr))->counter) \
	40	: "a" (ptr), "d" (op_val), \
	41	"m" (((atomic_t *)(ptr))->counter) \
	42	: "cc", "memory" ); \
	43	new_val; \
	44	})
	45	#define atomic_read(v) ((v)->counter)
	46	#define atomic_set(v,i) (((v)->counter) = (i))
	47
1da177e4 LT	48	static __inline__ int atomic_add_return(int i, atomic_t * v)
	49	{
	50	return __CS_LOOP(v, i, "ar");
	51	}
973bd993 MS	52	#define atomic_add(_i, _v) atomic_add_return(_i, _v)
	53	#define atomic_add_negative(_i, _v) (atomic_add_return(_i, _v) < 0)
	54	#define atomic_inc(_v) atomic_add_return(1, _v)
	55	#define atomic_inc_return(_v) atomic_add_return(1, _v)
	56	#define atomic_inc_and_test(_v) (atomic_add_return(1, _v) == 0)
	57
1da177e4 LT	58	static __inline__ int atomic_sub_return(int i, atomic_t * v)
	59	{
	60	return __CS_LOOP(v, i, "sr");
	61	}
973bd993 MS	62	#define atomic_sub(_i, _v) atomic_sub_return(_i, _v)
	63	#define atomic_sub_and_test(_i, _v) (atomic_sub_return(_i, _v) == 0)
	64	#define atomic_dec(_v) atomic_sub_return(1, _v)
	65	#define atomic_dec_return(_v) atomic_sub_return(1, _v)
	66	#define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0)
1da177e4	67
1da177e4 LT	68	static __inline__ void atomic_clear_mask(unsigned long mask, atomic_t * v)
	69	{
	70	__CS_LOOP(v, ~mask, "nr");
	71	}
973bd993	72
1da177e4 LT	73	static __inline__ void atomic_set_mask(unsigned long mask, atomic_t * v)
	74	{
	75	__CS_LOOP(v, mask, "or");
	76	}
973bd993	77
ffbf670f IM	78	#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
ffbf670f IM	79
973bd993 MS	80	static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new)
	81	{
	82	__asm__ __volatile__(" cs %0,%3,0(%2)\n"
	83	: "+d" (old), "=m" (v->counter)
	84	: "a" (v), "d" (new), "m" (v->counter)
	85	: "cc", "memory" );
	86	return old;
	87	}
	88
	89	static __inline__ int atomic_add_unless(atomic_t *v, int a, int u)
	90	{
	91	int c, old;
973bd993	92	c = atomic_read(v);
0b2fcfdb NP	93	for (;;) {
	94	if (unlikely(c == u))
	95	break;
	96	old = atomic_cmpxchg(v, c, c + a);
	97	if (likely(old == c))
	98	break;
973bd993	99	c = old;
0b2fcfdb	100	}
973bd993 MS	101	return c != u;
	102	}
	103
	104	#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
	105
1da177e4 LT	106	#undef __CS_LOOP
	107
	108	#ifdef __s390x__
	109	typedef struct {
	110	volatile long long counter;
	111	} __attribute__ ((aligned (8))) atomic64_t;
	112	#define ATOMIC64_INIT(i) { (i) }
	113
	114	#define __CSG_LOOP(ptr, op_val, op_string) ({ \
	115	typeof(ptr->counter) old_val, new_val; \
	116	__asm__ __volatile__(" lg %0,0(%3)\n" \
	117	"0: lgr %1,%0\n" \
	118	op_string " %1,%4\n" \
	119	" csg %0,%1,0(%3)\n" \
	120	" jl 0b" \
	121	: "=&d" (old_val), "=&d" (new_val), \
	122	"=m" (((atomic_t *)(ptr))->counter) \
	123	: "a" (ptr), "d" (op_val), \
	124	"m" (((atomic_t *)(ptr))->counter) \
	125	: "cc", "memory" ); \
	126	new_val; \
	127	})
	128	#define atomic64_read(v) ((v)->counter)
	129	#define atomic64_set(v,i) (((v)->counter) = (i))
	130
46ee058c	131	static __inline__ long long atomic64_add_return(long long i, atomic64_t * v)
1da177e4 LT	132	{
	133	return __CSG_LOOP(v, i, "agr");
	134	}
973bd993 MS	135	#define atomic64_add(_i, _v) atomic64_add_return(_i, _v)
	136	#define atomic64_add_negative(_i, _v) (atomic64_add_return(_i, _v) < 0)
	137	#define atomic64_inc(_v) atomic64_add_return(1, _v)
	138	#define atomic64_inc_return(_v) atomic64_add_return(1, _v)
	139	#define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0)
	140
	141	static __inline__ long long atomic64_sub_return(long long i, atomic64_t * v)
1da177e4	142	{
973bd993	143	return __CSG_LOOP(v, i, "sgr");
1da177e4	144	}
973bd993 MS	145	#define atomic64_sub(_i, _v) atomic64_sub_return(_i, _v)
	146	#define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0)
	147	#define atomic64_dec(_v) atomic64_sub_return(1, _v)
	148	#define atomic64_dec_return(_v) atomic64_sub_return(1, _v)
	149	#define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0)
	150
1da177e4 LT	151	static __inline__ void atomic64_clear_mask(unsigned long mask, atomic64_t * v)
	152	{
	153	__CSG_LOOP(v, ~mask, "ngr");
	154	}
973bd993	155
1da177e4 LT	156	static __inline__ void atomic64_set_mask(unsigned long mask, atomic64_t * v)
	157	{
	158	__CSG_LOOP(v, mask, "ogr");
	159	}
	160
973bd993 MS	161	static __inline__ long long atomic64_cmpxchg(atomic64_t *v,
	162	long long old, long long new)
	163	{
	164	__asm__ __volatile__(" csg %0,%3,0(%2)\n"
	165	: "+d" (old), "=m" (v->counter)
	166	: "a" (v), "d" (new), "m" (v->counter)
	167	: "cc", "memory" );
	168	return old;
	169	}
1da177e4	170
973bd993 MS	171	static __inline__ int atomic64_add_unless(atomic64_t *v,
973bd993 MS	172	long long a, long long u)
1da177e4	173	{
973bd993	174	long long c, old;
973bd993	175	c = atomic64_read(v);
0b2fcfdb NP	176	for (;;) {
	177	if (unlikely(c == u))
	178	break;
	179	old = atomic64_cmpxchg(v, c, c + a);
	180	if (likely(old == c))
	181	break;
973bd993	182	c = old;
0b2fcfdb	183	}
973bd993	184	return c != u;
1da177e4 LT	185	}
1da177e4 LT	186
973bd993	187	#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
4a6dae6d	188
973bd993 MS	189	#undef __CSG_LOOP
973bd993 MS	190	#endif
8426e1f6	191
1da177e4 LT	192	#define smp_mb__before_atomic_dec() smp_mb()
	193	#define smp_mb__after_atomic_dec() smp_mb()
	194	#define smp_mb__before_atomic_inc() smp_mb()
	195	#define smp_mb__after_atomic_inc() smp_mb()
	196
d3cb4871	197	#include <asm-generic/atomic.h>
1da177e4 LT	198	#endif /* __KERNEL__ */
1da177e4 LT	199	#endif /* __ARCH_S390_ATOMIC__ */