[linux-2.6-block.git] / arch / x86 / include / asm / mutex_64.h

/*
 * Assembly implementation of the mutex fastpath, based on atomic
 * decrement/increment.
 *
 * started by Ingo Molnar:
 *
 *  Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
 */
#ifndef _ASM_X86_MUTEX_64_H
#define _ASM_X86_MUTEX_64_H

/**
 * __mutex_fastpath_lock - decrement and call function if negative
 * @v: pointer of type atomic_t
 * @fail_fn: function to call if the result is negative
 *
 * Atomically decrements @v and calls <fail_fn> if the result is negative.
 */
#ifdef CC_HAVE_ASM_GOTO
static inline void __mutex_fastpath_lock(atomic_t *v,
					 void (*fail_fn)(atomic_t *))
{
	asm_volatile_goto(LOCK_PREFIX "   decl %0\n"
			  "   jns %l[exit]\n"
			  : : "m" (v->counter)
			  : "memory", "cc"
			  : exit);
	fail_fn(v);
exit:
	return;
}
#else
#define __mutex_fastpath_lock(v, fail_fn)			\
do {								\
	unsigned long dummy;					\
								\
	typecheck(atomic_t *, v);				\
	typecheck_fn(void (*)(atomic_t *), fail_fn);		\
								\
	asm volatile(LOCK_PREFIX "   decl (%%rdi)\n"		\
		     "   jns 1f		\n"			\
		     "   call " #fail_fn "\n"			\
		     "1:"					\
		     : "=D" (dummy)				\
		     : "D" (v)					\
		     : "rax", "rsi", "rdx", "rcx",		\
		       "r8", "r9", "r10", "r11", "memory");	\
} while (0)
#endif

/**
 *  __mutex_fastpath_lock_retval - try to take the lock by moving the count
 *                                 from 1 to a 0 value
 *  @count: pointer of type atomic_t
 *
 * Change the count from 1 to a value lower than 1. This function returns 0
 * if the fastpath succeeds, or -1 otherwise.
 */
static inline int __mutex_fastpath_lock_retval(atomic_t *count)
{
	if (unlikely(atomic_dec_return(count) < 0))
		return -1;
	else
		return 0;
}

/**
 * __mutex_fastpath_unlock - increment and call function if nonpositive
 * @v: pointer of type atomic_t
 * @fail_fn: function to call if the result is nonpositive
 *
 * Atomically increments @v and calls <fail_fn> if the result is nonpositive.
 */
#ifdef CC_HAVE_ASM_GOTO
static inline void __mutex_fastpath_unlock(atomic_t *v,
					   void (*fail_fn)(atomic_t *))
{
	asm_volatile_goto(LOCK_PREFIX "   incl %0\n"
			  "   jg %l[exit]\n"
			  : : "m" (v->counter)
			  : "memory", "cc"
			  : exit);
	fail_fn(v);
exit:
	return;
}
#else
#define __mutex_fastpath_unlock(v, fail_fn)			\
do {								\
	unsigned long dummy;					\
								\
	typecheck(atomic_t *, v);				\
	typecheck_fn(void (*)(atomic_t *), fail_fn);		\
								\
	asm volatile(LOCK_PREFIX "   incl (%%rdi)\n"		\
		     "   jg 1f\n"				\
		     "   call " #fail_fn "\n"			\
		     "1:"					\
		     : "=D" (dummy)				\
		     : "D" (v)					\
		     : "rax", "rsi", "rdx", "rcx",		\
		       "r8", "r9", "r10", "r11", "memory");	\
} while (0)
#endif

#define __mutex_slowpath_needs_to_unlock()	1

/**
 * __mutex_fastpath_trylock - try to acquire the mutex, without waiting
 *
 *  @count: pointer of type atomic_t
 *  @fail_fn: fallback function
 *
 * Change the count from 1 to 0 and return 1 (success), or return 0 (failure)
 * if it wasn't 1 originally. [the fallback function is never used on
 * x86_64, because all x86_64 CPUs have a CMPXCHG instruction.]
 */
static inline int __mutex_fastpath_trylock(atomic_t *count,
					   int (*fail_fn)(atomic_t *))
{
	if (likely(atomic_read(count) == 1 && atomic_cmpxchg(count, 1, 0) == 1))
		return 1;

	return 0;
}

#endif /* _ASM_X86_MUTEX_64_H */
Commit	Line	Data
b8aa0361 IM	1	/*
	2	* Assembly implementation of the mutex fastpath, based on atomic
	3	* decrement/increment.
	4	*
	5	* started by Ingo Molnar:
	6	*
	7	* Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
	8	*/
1965aae3 PA	9	#ifndef _ASM_X86_MUTEX_64_H
1965aae3 PA	10	#define _ASM_X86_MUTEX_64_H
b8aa0361 IM	11
	12	/**
	13	* __mutex_fastpath_lock - decrement and call function if negative
	14	* @v: pointer of type atomic_t
	15	* @fail_fn: function to call if the result is negative
	16	*
	17	* Atomically decrements @v and calls <fail_fn> if the result is negative.
	18	*/
00e55a79 WAF	19	#ifdef CC_HAVE_ASM_GOTO
	20	static inline void __mutex_fastpath_lock(atomic_t *v,
	21	void (fail_fn)(atomic_t ))
	22	{
3f0116c3	23	asm_volatile_goto(LOCK_PREFIX " decl %0\n"
00e55a79 WAF	24	" jns %l[exit]\n"
	25	: : "m" (v->counter)
	26	: "memory", "cc"
	27	: exit);
	28	fail_fn(v);
	29	exit:
	30	return;
	31	}
	32	#else
2c4e8830 JP	33	#define __mutex_fastpath_lock(v, fail_fn) \
	34	do { \
	35	unsigned long dummy; \
	36	\
	37	typecheck(atomic_t *, v); \
	38	typecheck_fn(void ()(atomic_t ), fail_fn); \
	39	\
	40	asm volatile(LOCK_PREFIX " decl (%%rdi)\n" \
	41	" jns 1f \n" \
	42	" call " #fail_fn "\n" \
	43	"1:" \
	44	: "=D" (dummy) \
	45	: "D" (v) \
	46	: "rax", "rsi", "rdx", "rcx", \
	47	"r8", "r9", "r10", "r11", "memory"); \
b8aa0361	48	} while (0)
00e55a79	49	#endif
b8aa0361 IM	50
	51	/**
	52	* __mutex_fastpath_lock_retval - try to take the lock by moving the count
	53	* from 1 to a 0 value
	54	* @count: pointer of type atomic_t
b8aa0361	55	*
a41b56ef ML	56	* Change the count from 1 to a value lower than 1. This function returns 0
a41b56ef ML	57	* if the fastpath succeeds, or -1 otherwise.
b8aa0361	58	*/
a41b56ef	59	static inline int __mutex_fastpath_lock_retval(atomic_t *count)
b8aa0361 IM	60	{
b8aa0361 IM	61	if (unlikely(atomic_dec_return(count) < 0))
a41b56ef	62	return -1;
b8aa0361 IM	63	else
	64	return 0;
	65	}
	66
	67	/**
	68	* __mutex_fastpath_unlock - increment and call function if nonpositive
	69	* @v: pointer of type atomic_t
	70	* @fail_fn: function to call if the result is nonpositive
	71	*
	72	* Atomically increments @v and calls <fail_fn> if the result is nonpositive.
	73	*/
00e55a79 WAF	74	#ifdef CC_HAVE_ASM_GOTO
	75	static inline void __mutex_fastpath_unlock(atomic_t *v,
	76	void (fail_fn)(atomic_t ))
	77	{
3f0116c3	78	asm_volatile_goto(LOCK_PREFIX " incl %0\n"
00e55a79 WAF	79	" jg %l[exit]\n"
	80	: : "m" (v->counter)
	81	: "memory", "cc"
	82	: exit);
	83	fail_fn(v);
	84	exit:
	85	return;
	86	}
	87	#else
2c4e8830 JP	88	#define __mutex_fastpath_unlock(v, fail_fn) \
	89	do { \
	90	unsigned long dummy; \
	91	\
	92	typecheck(atomic_t *, v); \
	93	typecheck_fn(void ()(atomic_t ), fail_fn); \
	94	\
	95	asm volatile(LOCK_PREFIX " incl (%%rdi)\n" \
	96	" jg 1f\n" \
	97	" call " #fail_fn "\n" \
	98	"1:" \
	99	: "=D" (dummy) \
	100	: "D" (v) \
	101	: "rax", "rsi", "rdx", "rcx", \
	102	"r8", "r9", "r10", "r11", "memory"); \
b8aa0361	103	} while (0)
00e55a79	104	#endif
b8aa0361 IM	105
	106	#define __mutex_slowpath_needs_to_unlock() 1
	107
	108	/**
	109	* __mutex_fastpath_trylock - try to acquire the mutex, without waiting
	110	*
	111	* @count: pointer of type atomic_t
	112	* @fail_fn: fallback function
	113	*
	114	* Change the count from 1 to 0 and return 1 (success), or return 0 (failure)
	115	* if it wasn't 1 originally. [the fallback function is never used on
	116	* x86_64, because all x86_64 CPUs have a CMPXCHG instruction.]
	117	*/
2c4e8830 JP	118	static inline int __mutex_fastpath_trylock(atomic_t *count,
2c4e8830 JP	119	int (fail_fn)(atomic_t ))
b8aa0361	120	{
6428671b	121	if (likely(atomic_read(count) == 1 && atomic_cmpxchg(count, 1, 0) == 1))
b8aa0361	122	return 1;
6428671b PZ	123
6428671b PZ	124	return 0;
b8aa0361 IM	125	}
b8aa0361 IM	126
1965aae3	127	#endif /* _ASM_X86_MUTEX_64_H */