[linux-2.6-block.git] / include / asm-arm / mutex.h

/*
 * include/asm-arm/mutex.h
 *
 * ARM optimized mutex locking primitives
 *
 * Please look into asm-generic/mutex-xchg.h for a formal definition.
 */
#ifndef _ASM_MUTEX_H
#define _ASM_MUTEX_H

#if __LINUX_ARM_ARCH__ < 6
/* On pre-ARMv6 hardware the swp based implementation is the most efficient. */
# include <asm-generic/mutex-xchg.h>
#else

/*
 * Attempting to lock a mutex on ARMv6+ can be done with a bastardized
 * atomic decrement (it is not a reliable atomic decrement but it satisfies
 * the defined semantics for our purpose, while being smaller and faster
 * than a real atomic decrement or atomic swap.  The idea is to attempt
 * decrementing the lock value only once.  If once decremented it isn't zero,
 * or if its store-back fails due to a dispute on the exclusive store, we
 * simply bail out immediately through the slow path where the lock will be
 * reattempted until it succeeds.
 */
static inline void
__mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *))
{
	int __ex_flag, __res;

	__asm__ (

		"ldrex	%0, [%2]	\n\t"
		"sub	%0, %0, #1	\n\t"
		"strex	%1, %0, [%2]	"

		: "=&r" (__res), "=&r" (__ex_flag)
		: "r" (&(count)->counter)
		: "cc","memory" );

	__res |= __ex_flag;
	if (unlikely(__res != 0))
		fail_fn(count);
}

static inline int
__mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *))
{
	int __ex_flag, __res;

	__asm__ (

		"ldrex	%0, [%2]	\n\t"
		"sub	%0, %0, #1	\n\t"
		"strex	%1, %0, [%2]	"

		: "=&r" (__res), "=&r" (__ex_flag)
		: "r" (&(count)->counter)
		: "cc","memory" );

	__res |= __ex_flag;
	if (unlikely(__res != 0))
		__res = fail_fn(count);
	return __res;
}

/*
 * Same trick is used for the unlock fast path. However the original value,
 * rather than the result, is used to test for success in order to have
 * better generated assembly.
 */
static inline void
__mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *))
{
	int __ex_flag, __res, __orig;

	__asm__ (

		"ldrex	%0, [%3]	\n\t"
		"add	%1, %0, #1	\n\t"
		"strex	%2, %1, [%3]	"

		: "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag)
		: "r" (&(count)->counter)
		: "cc","memory" );

	__orig |= __ex_flag;
	if (unlikely(__orig != 0))
		fail_fn(count);
}

/*
 * If the unlock was done on a contended lock, or if the unlock simply fails
 * then the mutex remains locked.
 */
#define __mutex_slowpath_needs_to_unlock()	1

/*
 * For __mutex_fastpath_trylock we use another construct which could be
 * described as a "single value cmpxchg".
 *
 * This provides the needed trylock semantics like cmpxchg would, but it is
 * lighter and less generic than a true cmpxchg implementation.
 */
static inline int
__mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *))
{
	int __ex_flag, __res, __orig;

	__asm__ (

		"1: ldrex	%0, [%3]	\n\t"
		"subs		%1, %0, #1	\n\t"
		"strexeq	%2, %1, [%3]	\n\t"
		"movlt		%0, #0		\n\t"
		"cmpeq		%2, #0		\n\t"
		"bgt		1b		"

		: "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag)
		: "r" (&count->counter)
		: "cc", "memory" );

	return __orig;
}

#endif
#endif
Commit	Line	Data
823d0f4f NP	1	/*
	2	* include/asm-arm/mutex.h
	3	*
	4	* ARM optimized mutex locking primitives
	5	*
	6	* Please look into asm-generic/mutex-xchg.h for a formal definition.
	7	*/
	8	#ifndef _ASM_MUTEX_H
	9	#define _ASM_MUTEX_H
	10
	11	#if __LINUX_ARM_ARCH__ < 6
	12	/* On pre-ARMv6 hardware the swp based implementation is the most efficient. */
	13	# include <asm-generic/mutex-xchg.h>
	14	#else
	15
	16	/*
	17	* Attempting to lock a mutex on ARMv6+ can be done with a bastardized
	18	* atomic decrement (it is not a reliable atomic decrement but it satisfies
	19	* the defined semantics for our purpose, while being smaller and faster
	20	* than a real atomic decrement or atomic swap. The idea is to attempt
	21	* decrementing the lock value only once. If once decremented it isn't zero,
	22	* or if its store-back fails due to a dispute on the exclusive store, we
	23	* simply bail out immediately through the slow path where the lock will be
	24	* reattempted until it succeeds.
	25	*/
365bf8ac	26	static inline void
edde08f2	27	__mutex_fastpath_lock(atomic_t count, void (fail_fn)(atomic_t *))
365bf8ac NP	28	{
	29	int __ex_flag, __res;
	30
	31	__asm__ (
	32
	33	"ldrex %0, [%2] \n\t"
	34	"sub %0, %0, #1 \n\t"
	35	"strex %1, %0, [%2] "
	36
	37	: "=&r" (__res), "=&r" (__ex_flag)
	38	: "r" (&(count)->counter)
	39	: "cc","memory" );
	40
	41	__res \|= __ex_flag;
	42	if (unlikely(__res != 0))
	43	fail_fn(count);
	44	}
	45
	46	static inline int
edde08f2	47	__mutex_fastpath_lock_retval(atomic_t count, int (fail_fn)(atomic_t *))
365bf8ac NP	48	{
	49	int __ex_flag, __res;
	50
	51	__asm__ (
	52
	53	"ldrex %0, [%2] \n\t"
	54	"sub %0, %0, #1 \n\t"
	55	"strex %1, %0, [%2] "
	56
	57	: "=&r" (__res), "=&r" (__ex_flag)
	58	: "r" (&(count)->counter)
	59	: "cc","memory" );
	60
	61	__res \|= __ex_flag;
	62	if (unlikely(__res != 0))
	63	__res = fail_fn(count);
	64	return __res;
	65	}
823d0f4f NP	66
	67	/*
	68	* Same trick is used for the unlock fast path. However the original value,
	69	* rather than the result, is used to test for success in order to have
	70	* better generated assembly.
	71	*/
365bf8ac	72	static inline void
edde08f2	73	__mutex_fastpath_unlock(atomic_t count, void (fail_fn)(atomic_t *))
365bf8ac NP	74	{
	75	int __ex_flag, __res, __orig;
	76
	77	__asm__ (
	78
	79	"ldrex %0, [%3] \n\t"
	80	"add %1, %0, #1 \n\t"
	81	"strex %2, %1, [%3] "
	82
	83	: "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag)
	84	: "r" (&(count)->counter)
	85	: "cc","memory" );
	86
	87	__orig \|= __ex_flag;
	88	if (unlikely(__orig != 0))
	89	fail_fn(count);
	90	}
823d0f4f NP	91
	92	/*
	93	* If the unlock was done on a contended lock, or if the unlock simply fails
	94	* then the mutex remains locked.
	95	*/
	96	#define __mutex_slowpath_needs_to_unlock() 1
	97
	98	/*
	99	* For __mutex_fastpath_trylock we use another construct which could be
	100	* described as a "single value cmpxchg".
	101	*
	102	* This provides the needed trylock semantics like cmpxchg would, but it is
	103	* lighter and less generic than a true cmpxchg implementation.
	104	*/
	105	static inline int
	106	__mutex_fastpath_trylock(atomic_t count, int (fail_fn)(atomic_t *))
	107	{
	108	int __ex_flag, __res, __orig;
	109
	110	__asm__ (
	111
365bf8ac NP	112	"1: ldrex %0, [%3] \n\t"
	113	"subs %1, %0, #1 \n\t"
	114	"strexeq %2, %1, [%3] \n\t"
	115	"movlt %0, #0 \n\t"
	116	"cmpeq %2, #0 \n\t"
	117	"bgt 1b "
823d0f4f NP	118
	119	: "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag)
	120	: "r" (&count->counter)
	121	: "cc", "memory" );
	122
	123	return __orig;
	124	}
	125
	126	#endif
	127	#endif