[linux-2.6-block.git] / lib / lockref.c

#include <linux/export.h>
#include <linux/lockref.h>

#if USE_CMPXCHG_LOCKREF

/*
 * Allow weakly-ordered memory architectures to provide barrier-less
 * cmpxchg semantics for lockref updates.
 */
#ifndef cmpxchg64_relaxed
# define cmpxchg64_relaxed cmpxchg64
#endif

/*
 * Note that the "cmpxchg()" reloads the "old" value for the
 * failure case.
 */
#define CMPXCHG_LOOP(CODE, SUCCESS) do {					\
	struct lockref old;							\
	BUILD_BUG_ON(sizeof(old) != 8);						\
	old.lock_count = READ_ONCE(lockref->lock_count);			\
	while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) {  	\
		struct lockref new = old, prev = old;				\
		CODE								\
		old.lock_count = cmpxchg64_relaxed(&lockref->lock_count,	\
						   old.lock_count,		\
						   new.lock_count);		\
		if (likely(old.lock_count == prev.lock_count)) {		\
			SUCCESS;						\
		}								\
		cpu_relax_lowlatency();						\
	}									\
} while (0)

#else

#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)

#endif

/**
 * lockref_get - Increments reference count unconditionally
 * @lockref: pointer to lockref structure
 *
 * This operation is only valid if you already hold a reference
 * to the object, so you know the count cannot be zero.
 */
void lockref_get(struct lockref *lockref)
{
	CMPXCHG_LOOP(
		new.count++;
	,
		return;
	);

	spin_lock(&lockref->lock);
	lockref->count++;
	spin_unlock(&lockref->lock);
}
EXPORT_SYMBOL(lockref_get);

/**
 * lockref_get_not_zero - Increments count unless the count is 0 or dead
 * @lockref: pointer to lockref structure
 * Return: 1 if count updated successfully or 0 if count was zero
 */
int lockref_get_not_zero(struct lockref *lockref)
{
	int retval;

	CMPXCHG_LOOP(
		new.count++;
		if (old.count <= 0)
			return 0;
	,
		return 1;
	);

	spin_lock(&lockref->lock);
	retval = 0;
	if (lockref->count > 0) {
		lockref->count++;
		retval = 1;
	}
	spin_unlock(&lockref->lock);
	return retval;
}
EXPORT_SYMBOL(lockref_get_not_zero);

/**
 * lockref_get_or_lock - Increments count unless the count is 0 or dead
 * @lockref: pointer to lockref structure
 * Return: 1 if count updated successfully or 0 if count was zero
 * and we got the lock instead.
 */
int lockref_get_or_lock(struct lockref *lockref)
{
	CMPXCHG_LOOP(
		new.count++;
		if (old.count <= 0)
			break;
	,
		return 1;
	);

	spin_lock(&lockref->lock);
	if (lockref->count <= 0)
		return 0;
	lockref->count++;
	spin_unlock(&lockref->lock);
	return 1;
}
EXPORT_SYMBOL(lockref_get_or_lock);

/**
 * lockref_put_return - Decrement reference count if possible
 * @lockref: pointer to lockref structure
 *
 * Decrement the reference count and return the new value.
 * If the lockref was dead or locked, return an error.
 */
int lockref_put_return(struct lockref *lockref)
{
	CMPXCHG_LOOP(
		new.count--;
		if (old.count <= 0)
			return -1;
	,
		return new.count;
	);
	return -1;
}
EXPORT_SYMBOL(lockref_put_return);

/**
 * lockref_put_or_lock - decrements count unless count <= 1 before decrement
 * @lockref: pointer to lockref structure
 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
 */
int lockref_put_or_lock(struct lockref *lockref)
{
	CMPXCHG_LOOP(
		new.count--;
		if (old.count <= 1)
			break;
	,
		return 1;
	);

	spin_lock(&lockref->lock);
	if (lockref->count <= 1)
		return 0;
	lockref->count--;
	spin_unlock(&lockref->lock);
	return 1;
}
EXPORT_SYMBOL(lockref_put_or_lock);

/**
 * lockref_mark_dead - mark lockref dead
 * @lockref: pointer to lockref structure
 */
void lockref_mark_dead(struct lockref *lockref)
{
	assert_spin_locked(&lockref->lock);
	lockref->count = -128;
}
EXPORT_SYMBOL(lockref_mark_dead);

/**
 * lockref_get_not_dead - Increments count unless the ref is dead
 * @lockref: pointer to lockref structure
 * Return: 1 if count updated successfully or 0 if lockref was dead
 */
int lockref_get_not_dead(struct lockref *lockref)
{
	int retval;

	CMPXCHG_LOOP(
		new.count++;
		if (old.count < 0)
			return 0;
	,
		return 1;
	);

	spin_lock(&lockref->lock);
	retval = 0;
	if (lockref->count >= 0) {
		lockref->count++;
		retval = 1;
	}
	spin_unlock(&lockref->lock);
	return retval;
}
EXPORT_SYMBOL(lockref_get_not_dead);
Commit	Line	Data
2f4f12e5 LT	1	#include <linux/export.h>
	2	#include <linux/lockref.h>
	3
57f4257e	4	#if USE_CMPXCHG_LOCKREF
bc08b449	5
d2212b4d WD	6	/*
	7	* Allow weakly-ordered memory architectures to provide barrier-less
	8	* cmpxchg semantics for lockref updates.
	9	*/
	10	#ifndef cmpxchg64_relaxed
	11	# define cmpxchg64_relaxed cmpxchg64
	12	#endif
	13
bc08b449 LT	14	/*
	15	* Note that the "cmpxchg()" reloads the "old" value for the
	16	* failure case.
	17	*/
	18	#define CMPXCHG_LOOP(CODE, SUCCESS) do { \
	19	struct lockref old; \
	20	BUILD_BUG_ON(sizeof(old) != 8); \
4d3199e4	21	old.lock_count = READ_ONCE(lockref->lock_count); \
bc08b449 LT	22	while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
	23	struct lockref new = old, prev = old; \
	24	CODE \
d2212b4d WD	25	old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \
	26	old.lock_count, \
	27	new.lock_count); \
bc08b449 LT	28	if (likely(old.lock_count == prev.lock_count)) { \
	29	SUCCESS; \
	30	} \
3a6bfbc9	31	cpu_relax_lowlatency(); \
bc08b449 LT	32	} \
	33	} while (0)
	34
	35	#else
	36
	37	#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
	38
	39	#endif
	40
2f4f12e5 LT	41	/**
2f4f12e5 LT	42	* lockref_get - Increments reference count unconditionally
44a0cf92	43	* @lockref: pointer to lockref structure
2f4f12e5 LT	44	*
	45	* This operation is only valid if you already hold a reference
	46	* to the object, so you know the count cannot be zero.
	47	*/
	48	void lockref_get(struct lockref *lockref)
	49	{
bc08b449 LT	50	CMPXCHG_LOOP(
	51	new.count++;
	52	,
	53	return;
	54	);
	55
2f4f12e5 LT	56	spin_lock(&lockref->lock);
	57	lockref->count++;
	58	spin_unlock(&lockref->lock);
	59	}
	60	EXPORT_SYMBOL(lockref_get);
	61
	62	/**
360f5479	63	* lockref_get_not_zero - Increments count unless the count is 0 or dead
44a0cf92	64	* @lockref: pointer to lockref structure
2f4f12e5 LT	65	* Return: 1 if count updated successfully or 0 if count was zero
	66	*/
	67	int lockref_get_not_zero(struct lockref *lockref)
	68	{
bc08b449 LT	69	int retval;
	70
	71	CMPXCHG_LOOP(
	72	new.count++;
360f5479	73	if (old.count <= 0)
bc08b449 LT	74	return 0;
	75	,
	76	return 1;
	77	);
2f4f12e5 LT	78
2f4f12e5 LT	79	spin_lock(&lockref->lock);
bc08b449	80	retval = 0;
360f5479	81	if (lockref->count > 0) {
2f4f12e5 LT	82	lockref->count++;
	83	retval = 1;
	84	}
	85	spin_unlock(&lockref->lock);
	86	return retval;
	87	}
	88	EXPORT_SYMBOL(lockref_get_not_zero);
	89
	90	/**
360f5479	91	* lockref_get_or_lock - Increments count unless the count is 0 or dead
44a0cf92	92	* @lockref: pointer to lockref structure
2f4f12e5 LT	93	* Return: 1 if count updated successfully or 0 if count was zero
	94	* and we got the lock instead.
	95	*/
	96	int lockref_get_or_lock(struct lockref *lockref)
	97	{
bc08b449 LT	98	CMPXCHG_LOOP(
bc08b449 LT	99	new.count++;
360f5479	100	if (old.count <= 0)
bc08b449 LT	101	break;
	102	,
	103	return 1;
	104	);
	105
2f4f12e5	106	spin_lock(&lockref->lock);
360f5479	107	if (lockref->count <= 0)
2f4f12e5 LT	108	return 0;
	109	lockref->count++;
	110	spin_unlock(&lockref->lock);
	111	return 1;
	112	}
	113	EXPORT_SYMBOL(lockref_get_or_lock);
	114
360f5479 LT	115	/**
	116	* lockref_put_return - Decrement reference count if possible
	117	* @lockref: pointer to lockref structure
	118	*
	119	* Decrement the reference count and return the new value.
	120	* If the lockref was dead or locked, return an error.
	121	*/
	122	int lockref_put_return(struct lockref *lockref)
	123	{
	124	CMPXCHG_LOOP(
	125	new.count--;
	126	if (old.count <= 0)
	127	return -1;
	128	,
	129	return new.count;
	130	);
	131	return -1;
	132	}
	133	EXPORT_SYMBOL(lockref_put_return);
	134
2f4f12e5 LT	135	/**
2f4f12e5 LT	136	* lockref_put_or_lock - decrements count unless count <= 1 before decrement
44a0cf92	137	* @lockref: pointer to lockref structure
2f4f12e5 LT	138	* Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
	139	*/
	140	int lockref_put_or_lock(struct lockref *lockref)
	141	{
bc08b449 LT	142	CMPXCHG_LOOP(
	143	new.count--;
	144	if (old.count <= 1)
	145	break;
	146	,
	147	return 1;
	148	);
	149
2f4f12e5 LT	150	spin_lock(&lockref->lock);
	151	if (lockref->count <= 1)
	152	return 0;
	153	lockref->count--;
	154	spin_unlock(&lockref->lock);
	155	return 1;
	156	}
	157	EXPORT_SYMBOL(lockref_put_or_lock);
e7d33bb5 LT	158
	159	/**
	160	* lockref_mark_dead - mark lockref dead
	161	* @lockref: pointer to lockref structure
	162	*/
	163	void lockref_mark_dead(struct lockref *lockref)
	164	{
	165	assert_spin_locked(&lockref->lock);
	166	lockref->count = -128;
	167	}
e66cf161	168	EXPORT_SYMBOL(lockref_mark_dead);
e7d33bb5 LT	169
	170	/**
	171	* lockref_get_not_dead - Increments count unless the ref is dead
	172	* @lockref: pointer to lockref structure
	173	* Return: 1 if count updated successfully or 0 if lockref was dead
	174	*/
	175	int lockref_get_not_dead(struct lockref *lockref)
	176	{
	177	int retval;
	178
	179	CMPXCHG_LOOP(
	180	new.count++;
360f5479	181	if (old.count < 0)
e7d33bb5 LT	182	return 0;
	183	,
	184	return 1;
	185	);
	186
	187	spin_lock(&lockref->lock);
	188	retval = 0;
360f5479	189	if (lockref->count >= 0) {
e7d33bb5 LT	190	lockref->count++;
	191	retval = 1;
	192	}
	193	spin_unlock(&lockref->lock);
	194	return retval;
	195	}
	196	EXPORT_SYMBOL(lockref_get_not_dead);