[linux-2.6-block.git] / include / linux / percpu-refcount.h

/*
 * Percpu refcounts:
 * (C) 2012 Google, Inc.
 * Author: Kent Overstreet <koverstreet@google.com>
 *
 * This implements a refcount with similar semantics to atomic_t - atomic_inc(),
 * atomic_dec_and_test() - but percpu.
 *
 * There's one important difference between percpu refs and normal atomic_t
 * refcounts; you have to keep track of your initial refcount, and then when you
 * start shutting down you call percpu_ref_kill() _before_ dropping the initial
 * refcount.
 *
 * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less
 * than an atomic_t - this is because of the way shutdown works, see
 * percpu_ref_kill()/PERCPU_COUNT_BIAS.
 *
 * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the
 * refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill()
 * puts the ref back in single atomic_t mode, collecting the per cpu refs and
 * issuing the appropriate barriers, and then marks the ref as shutting down so
 * that percpu_ref_put() will check for the ref hitting 0.  After it returns,
 * it's safe to drop the initial ref.
 *
 * USAGE:
 *
 * See fs/aio.c for some example usage; it's used there for struct kioctx, which
 * is created when userspaces calls io_setup(), and destroyed when userspace
 * calls io_destroy() or the process exits.
 *
 * In the aio code, kill_ioctx() is called when we wish to destroy a kioctx; it
 * calls percpu_ref_kill(), then hlist_del_rcu() and synchronize_rcu() to remove
 * the kioctx from the proccess's list of kioctxs - after that, there can't be
 * any new users of the kioctx (from lookup_ioctx()) and it's then safe to drop
 * the initial ref with percpu_ref_put().
 *
 * Code that does a two stage shutdown like this often needs some kind of
 * explicit synchronization to ensure the initial refcount can only be dropped
 * once - percpu_ref_kill() does this for you, it returns true once and false if
 * someone else already called it. The aio code uses it this way, but it's not
 * necessary if the code has some other mechanism to synchronize teardown.
 * around.
 */

#ifndef _LINUX_PERCPU_REFCOUNT_H
#define _LINUX_PERCPU_REFCOUNT_H

#include <linux/atomic.h>
#include <linux/kernel.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/gfp.h>

struct percpu_ref;
typedef void (percpu_ref_func_t)(struct percpu_ref *);

/* flags set in the lower bits of percpu_ref->percpu_count_ptr */
enum {
	__PERCPU_REF_ATOMIC	= 1LU << 0,	/* operating in atomic mode */
	__PERCPU_REF_DEAD	= 1LU << 1,	/* (being) killed */
	__PERCPU_REF_ATOMIC_DEAD = __PERCPU_REF_ATOMIC | __PERCPU_REF_DEAD,

	__PERCPU_REF_FLAG_BITS	= 2,
};

/* @flags for percpu_ref_init() */
enum {
	/*
	 * Start w/ ref == 1 in atomic mode.  Can be switched to percpu
	 * operation using percpu_ref_switch_to_percpu().  If initialized
	 * with this flag, the ref will stay in atomic mode until
	 * percpu_ref_switch_to_percpu() is invoked on it.
	 */
	PERCPU_REF_INIT_ATOMIC	= 1 << 0,

	/*
	 * Start dead w/ ref == 0 in atomic mode.  Must be revived with
	 * percpu_ref_reinit() before used.  Implies INIT_ATOMIC.
	 */
	PERCPU_REF_INIT_DEAD	= 1 << 1,
};

struct percpu_ref {
	atomic_long_t		count;
	/*
	 * The low bit of the pointer indicates whether the ref is in percpu
	 * mode; if set, then get/put will manipulate the atomic_t.
	 */
	unsigned long		percpu_count_ptr;
	percpu_ref_func_t	*release;
	percpu_ref_func_t	*confirm_switch;
	bool			force_atomic:1;
	struct rcu_head		rcu;
};

int __must_check percpu_ref_init(struct percpu_ref *ref,
				 percpu_ref_func_t *release, unsigned int flags,
				 gfp_t gfp);
void percpu_ref_exit(struct percpu_ref *ref);
void percpu_ref_switch_to_atomic(struct percpu_ref *ref,
				 percpu_ref_func_t *confirm_switch);
void percpu_ref_switch_to_percpu(struct percpu_ref *ref);
void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
				 percpu_ref_func_t *confirm_kill);
void percpu_ref_reinit(struct percpu_ref *ref);

/**
 * percpu_ref_kill - drop the initial ref
 * @ref: percpu_ref to kill
 *
 * Must be used to drop the initial ref on a percpu refcount; must be called
 * precisely once before shutdown.
 *
 * Puts @ref in non percpu mode, then does a call_rcu() before gathering up the
 * percpu counters and dropping the initial ref.
 */
static inline void percpu_ref_kill(struct percpu_ref *ref)
{
	percpu_ref_kill_and_confirm(ref, NULL);
}

/*
 * Internal helper.  Don't use outside percpu-refcount proper.  The
 * function doesn't return the pointer and let the caller test it for NULL
 * because doing so forces the compiler to generate two conditional
 * branches as it can't assume that @ref->percpu_count is not NULL.
 */
static inline bool __ref_is_percpu(struct percpu_ref *ref,
					  unsigned long __percpu **percpu_countp)
{
	unsigned long percpu_ptr;

	/*
	 * The value of @ref->percpu_count_ptr is tested for
	 * !__PERCPU_REF_ATOMIC, which may be set asynchronously, and then
	 * used as a pointer.  If the compiler generates a separate fetch
	 * when using it as a pointer, __PERCPU_REF_ATOMIC may be set in
	 * between contaminating the pointer value, meaning that
	 * ACCESS_ONCE() is required when fetching it.
	 *
	 * Also, we need a data dependency barrier to be paired with
	 * smp_store_release() in __percpu_ref_switch_to_percpu().
	 *
	 * Use lockless deref which contains both.
	 */
	percpu_ptr = lockless_dereference(ref->percpu_count_ptr);

	/*
	 * Theoretically, the following could test just ATOMIC; however,
	 * then we'd have to mask off DEAD separately as DEAD may be
	 * visible without ATOMIC if we race with percpu_ref_kill().  DEAD
	 * implies ATOMIC anyway.  Test them together.
	 */
	if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD))
		return false;

	*percpu_countp = (unsigned long __percpu *)percpu_ptr;
	return true;
}

/**
 * percpu_ref_get_many - increment a percpu refcount
 * @ref: percpu_ref to get
 * @nr: number of references to get
 *
 * Analogous to atomic_long_add().
 *
 * This function is safe to call as long as @ref is between init and exit.
 */
static inline void percpu_ref_get_many(struct percpu_ref *ref, unsigned long nr)
{
	unsigned long __percpu *percpu_count;

	rcu_read_lock_sched();

	if (__ref_is_percpu(ref, &percpu_count))
		this_cpu_add(*percpu_count, nr);
	else
		atomic_long_add(nr, &ref->count);

	rcu_read_unlock_sched();
}

/**
 * percpu_ref_get - increment a percpu refcount
 * @ref: percpu_ref to get
 *
 * Analagous to atomic_long_inc().
 *
 * This function is safe to call as long as @ref is between init and exit.
 */
static inline void percpu_ref_get(struct percpu_ref *ref)
{
	percpu_ref_get_many(ref, 1);
}

/**
 * percpu_ref_tryget - try to increment a percpu refcount
 * @ref: percpu_ref to try-get
 *
 * Increment a percpu refcount unless its count already reached zero.
 * Returns %true on success; %false on failure.
 *
 * This function is safe to call as long as @ref is between init and exit.
 */
static inline bool percpu_ref_tryget(struct percpu_ref *ref)
{
	unsigned long __percpu *percpu_count;
	int ret;

	rcu_read_lock_sched();

	if (__ref_is_percpu(ref, &percpu_count)) {
		this_cpu_inc(*percpu_count);
		ret = true;
	} else {
		ret = atomic_long_inc_not_zero(&ref->count);
	}

	rcu_read_unlock_sched();

	return ret;
}

/**
 * percpu_ref_tryget_live - try to increment a live percpu refcount
 * @ref: percpu_ref to try-get
 *
 * Increment a percpu refcount unless it has already been killed.  Returns
 * %true on success; %false on failure.
 *
 * Completion of percpu_ref_kill() in itself doesn't guarantee that this
 * function will fail.  For such guarantee, percpu_ref_kill_and_confirm()
 * should be used.  After the confirm_kill callback is invoked, it's
 * guaranteed that no new reference will be given out by
 * percpu_ref_tryget_live().
 *
 * This function is safe to call as long as @ref is between init and exit.
 */
static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
{
	unsigned long __percpu *percpu_count;
	int ret = false;

	rcu_read_lock_sched();

	if (__ref_is_percpu(ref, &percpu_count)) {
		this_cpu_inc(*percpu_count);
		ret = true;
	} else if (!(ref->percpu_count_ptr & __PERCPU_REF_DEAD)) {
		ret = atomic_long_inc_not_zero(&ref->count);
	}

	rcu_read_unlock_sched();

	return ret;
}

/**
 * percpu_ref_put_many - decrement a percpu refcount
 * @ref: percpu_ref to put
 * @nr: number of references to put
 *
 * Decrement the refcount, and if 0, call the release function (which was passed
 * to percpu_ref_init())
 *
 * This function is safe to call as long as @ref is between init and exit.
 */
static inline void percpu_ref_put_many(struct percpu_ref *ref, unsigned long nr)
{
	unsigned long __percpu *percpu_count;

	rcu_read_lock_sched();

	if (__ref_is_percpu(ref, &percpu_count))
		this_cpu_sub(*percpu_count, nr);
	else if (unlikely(atomic_long_sub_and_test(nr, &ref->count)))
		ref->release(ref);

	rcu_read_unlock_sched();
}

/**
 * percpu_ref_put - decrement a percpu refcount
 * @ref: percpu_ref to put
 *
 * Decrement the refcount, and if 0, call the release function (which was passed
 * to percpu_ref_init())
 *
 * This function is safe to call as long as @ref is between init and exit.
 */
static inline void percpu_ref_put(struct percpu_ref *ref)
{
	percpu_ref_put_many(ref, 1);
}

/**
 * percpu_ref_is_dying - test whether a percpu refcount is dying or dead
 * @ref: percpu_ref to test
 *
 * Returns %true if @ref is dying or dead.
 *
 * This function is safe to call as long as @ref is between init and exit
 * and the caller is responsible for synchronizing against state changes.
 */
static inline bool percpu_ref_is_dying(struct percpu_ref *ref)
{
	return ref->percpu_count_ptr & __PERCPU_REF_DEAD;
}

/**
 * percpu_ref_is_zero - test whether a percpu refcount reached zero
 * @ref: percpu_ref to test
 *
 * Returns %true if @ref reached zero.
 *
 * This function is safe to call as long as @ref is between init and exit.
 */
static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
{
	unsigned long __percpu *percpu_count;

	if (__ref_is_percpu(ref, &percpu_count))
		return false;
	return !atomic_long_read(&ref->count);
}

#endif
Commit	Line	Data
215e262f KO	1	/*
	2	* Percpu refcounts:
	3	* (C) 2012 Google, Inc.
	4	* Author: Kent Overstreet <koverstreet@google.com>
	5	*
	6	* This implements a refcount with similar semantics to atomic_t - atomic_inc(),
	7	* atomic_dec_and_test() - but percpu.
	8	*
	9	* There's one important difference between percpu refs and normal atomic_t
	10	* refcounts; you have to keep track of your initial refcount, and then when you
	11	* start shutting down you call percpu_ref_kill() _before_ dropping the initial
	12	* refcount.
	13	*
	14	* The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less
	15	* than an atomic_t - this is because of the way shutdown works, see
eecc16ba	16	* percpu_ref_kill()/PERCPU_COUNT_BIAS.
215e262f KO	17	*
	18	* Before you call percpu_ref_kill(), percpu_ref_put() does not check for the
	19	* refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill()
	20	* puts the ref back in single atomic_t mode, collecting the per cpu refs and
	21	* issuing the appropriate barriers, and then marks the ref as shutting down so
	22	* that percpu_ref_put() will check for the ref hitting 0. After it returns,
	23	* it's safe to drop the initial ref.
	24	*
	25	* USAGE:
	26	*
	27	* See fs/aio.c for some example usage; it's used there for struct kioctx, which
	28	* is created when userspaces calls io_setup(), and destroyed when userspace
	29	* calls io_destroy() or the process exits.
	30	*
	31	* In the aio code, kill_ioctx() is called when we wish to destroy a kioctx; it
d7cdb968	32	* calls percpu_ref_kill(), then hlist_del_rcu() and synchronize_rcu() to remove
215e262f KO	33	* the kioctx from the proccess's list of kioctxs - after that, there can't be
	34	* any new users of the kioctx (from lookup_ioctx()) and it's then safe to drop
	35	* the initial ref with percpu_ref_put().
	36	*
	37	* Code that does a two stage shutdown like this often needs some kind of
	38	* explicit synchronization to ensure the initial refcount can only be dropped
	39	* once - percpu_ref_kill() does this for you, it returns true once and false if
	40	* someone else already called it. The aio code uses it this way, but it's not
	41	* necessary if the code has some other mechanism to synchronize teardown.
	42	* around.
	43	*/
	44
	45	#ifndef _LINUX_PERCPU_REFCOUNT_H
	46	#define _LINUX_PERCPU_REFCOUNT_H
	47
	48	#include <linux/atomic.h>
	49	#include <linux/kernel.h>
	50	#include <linux/percpu.h>
	51	#include <linux/rcupdate.h>
a34375ef	52	#include <linux/gfp.h>
215e262f KO	53
215e262f KO	54	struct percpu_ref;
ac899061	55	typedef void (percpu_ref_func_t)(struct percpu_ref *);
215e262f	56
9e804d1f TH	57	/* flags set in the lower bits of percpu_ref->percpu_count_ptr */
	58	enum {
	59	__PERCPU_REF_ATOMIC = 1LU << 0, /* operating in atomic mode */
27344a90 TH	60	__PERCPU_REF_DEAD = 1LU << 1, /* (being) killed */
	61	__PERCPU_REF_ATOMIC_DEAD = __PERCPU_REF_ATOMIC \| __PERCPU_REF_DEAD,
	62
	63	__PERCPU_REF_FLAG_BITS = 2,
9e804d1f TH	64	};
9e804d1f TH	65
2aad2a86 TH	66	/* @flags for percpu_ref_init() */
	67	enum {
	68	/*
	69	* Start w/ ref == 1 in atomic mode. Can be switched to percpu
1cae13e7 TH	70	* operation using percpu_ref_switch_to_percpu(). If initialized
	71	* with this flag, the ref will stay in atomic mode until
	72	* percpu_ref_switch_to_percpu() is invoked on it.
2aad2a86 TH	73	*/
	74	PERCPU_REF_INIT_ATOMIC = 1 << 0,
	75
	76	/*
	77	* Start dead w/ ref == 0 in atomic mode. Must be revived with
	78	* percpu_ref_reinit() before used. Implies INIT_ATOMIC.
	79	*/
	80	PERCPU_REF_INIT_DEAD = 1 << 1,
	81	};
	82
215e262f	83	struct percpu_ref {
e625305b	84	atomic_long_t count;
215e262f KO	85	/*
215e262f KO	86	* The low bit of the pointer indicates whether the ref is in percpu
9a1049da	87	* mode; if set, then get/put will manipulate the atomic_t.
215e262f	88	*/
eecc16ba	89	unsigned long percpu_count_ptr;
ac899061	90	percpu_ref_func_t *release;
9e804d1f	91	percpu_ref_func_t *confirm_switch;
1cae13e7	92	bool force_atomic:1;
215e262f KO	93	struct rcu_head rcu;
	94	};
	95
acac7883	96	int __must_check percpu_ref_init(struct percpu_ref *ref,
2aad2a86 TH	97	percpu_ref_func_t *release, unsigned int flags,
2aad2a86 TH	98	gfp_t gfp);
9a1049da	99	void percpu_ref_exit(struct percpu_ref *ref);
490c79a6 TH	100	void percpu_ref_switch_to_atomic(struct percpu_ref *ref,
490c79a6 TH	101	percpu_ref_func_t *confirm_switch);
f47ad457	102	void percpu_ref_switch_to_percpu(struct percpu_ref *ref);
dbece3a0 TH	103	void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
dbece3a0 TH	104	percpu_ref_func_t *confirm_kill);
f47ad457	105	void percpu_ref_reinit(struct percpu_ref *ref);
dbece3a0 TH	106
	107	/**
	108	* percpu_ref_kill - drop the initial ref
	109	* @ref: percpu_ref to kill
	110	*
	111	* Must be used to drop the initial ref on a percpu refcount; must be called
	112	* precisely once before shutdown.
	113	*
	114	* Puts @ref in non percpu mode, then does a call_rcu() before gathering up the
	115	* percpu counters and dropping the initial ref.
	116	*/
	117	static inline void percpu_ref_kill(struct percpu_ref *ref)
	118	{
4d414269	119	percpu_ref_kill_and_confirm(ref, NULL);
dbece3a0	120	}
215e262f	121
eae7975d TH	122	/*
	123	* Internal helper. Don't use outside percpu-refcount proper. The
	124	* function doesn't return the pointer and let the caller test it for NULL
	125	* because doing so forces the compiler to generate two conditional
eecc16ba	126	* branches as it can't assume that @ref->percpu_count is not NULL.
eae7975d	127	*/
9e804d1f TH	128	static inline bool __ref_is_percpu(struct percpu_ref *ref,
9e804d1f TH	129	unsigned long __percpu **percpu_countp)
eae7975d	130	{
6810e4a3 TH	131	unsigned long percpu_ptr;
	132
	133	/*
	134	* The value of @ref->percpu_count_ptr is tested for
	135	* !__PERCPU_REF_ATOMIC, which may be set asynchronously, and then
	136	* used as a pointer. If the compiler generates a separate fetch
	137	* when using it as a pointer, __PERCPU_REF_ATOMIC may be set in
	138	* between contaminating the pointer value, meaning that
	139	* ACCESS_ONCE() is required when fetching it.
	140	*
	141	* Also, we need a data dependency barrier to be paired with
	142	* smp_store_release() in __percpu_ref_switch_to_percpu().
	143	*
	144	* Use lockless deref which contains both.
	145	*/
	146	percpu_ptr = lockless_dereference(ref->percpu_count_ptr);
2d722782	147
4aab3b5b TH	148	/*
	149	* Theoretically, the following could test just ATOMIC; however,
	150	* then we'd have to mask off DEAD separately as DEAD may be
	151	* visible without ATOMIC if we race with percpu_ref_kill(). DEAD
	152	* implies ATOMIC anyway. Test them together.
	153	*/
	154	if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD))
eae7975d TH	155	return false;
eae7975d TH	156
eecc16ba	157	percpu_countp = (unsigned long __percpu )percpu_ptr;
eae7975d TH	158	return true;
eae7975d TH	159	}
215e262f KO	160
215e262f KO	161	/**
e8ea14cc	162	* percpu_ref_get_many - increment a percpu refcount
ac899061	163	* @ref: percpu_ref to get
e8ea14cc	164	* @nr: number of references to get
215e262f	165	*
e8ea14cc	166	* Analogous to atomic_long_add().
6251f997 TH	167	*
	168	* This function is safe to call as long as @ref is between init and exit.
	169	*/
e8ea14cc	170	static inline void percpu_ref_get_many(struct percpu_ref *ref, unsigned long nr)
215e262f	171	{
eecc16ba	172	unsigned long __percpu *percpu_count;
215e262f	173
a4244454	174	rcu_read_lock_sched();
215e262f	175
9e804d1f	176	if (__ref_is_percpu(ref, &percpu_count))
e8ea14cc	177	this_cpu_add(*percpu_count, nr);
215e262f	178	else
e8ea14cc	179	atomic_long_add(nr, &ref->count);
215e262f	180
a4244454	181	rcu_read_unlock_sched();
215e262f KO	182	}
215e262f KO	183
e8ea14cc JW	184	/**
	185	* percpu_ref_get - increment a percpu refcount
	186	* @ref: percpu_ref to get
	187	*
	188	* Analagous to atomic_long_inc().
	189	*
	190	* This function is safe to call as long as @ref is between init and exit.
	191	*/
	192	static inline void percpu_ref_get(struct percpu_ref *ref)
	193	{
	194	percpu_ref_get_many(ref, 1);
	195	}
	196
4fb6e250 TH	197	/**
	198	* percpu_ref_tryget - try to increment a percpu refcount
	199	* @ref: percpu_ref to try-get
	200	*
	201	* Increment a percpu refcount unless its count already reached zero.
	202	* Returns %true on success; %false on failure.
	203	*
6251f997	204	* This function is safe to call as long as @ref is between init and exit.
4fb6e250 TH	205	*/
	206	static inline bool percpu_ref_tryget(struct percpu_ref *ref)
	207	{
eecc16ba	208	unsigned long __percpu *percpu_count;
6251f997	209	int ret;
4fb6e250 TH	210
	211	rcu_read_lock_sched();
	212
9e804d1f	213	if (__ref_is_percpu(ref, &percpu_count)) {
eecc16ba	214	this_cpu_inc(*percpu_count);
4fb6e250 TH	215	ret = true;
4fb6e250 TH	216	} else {
e625305b	217	ret = atomic_long_inc_not_zero(&ref->count);
4fb6e250 TH	218	}
	219
	220	rcu_read_unlock_sched();
	221
	222	return ret;
	223	}
	224
dbece3a0	225	/**
2070d50e	226	* percpu_ref_tryget_live - try to increment a live percpu refcount
dbece3a0 TH	227	* @ref: percpu_ref to try-get
	228	*
	229	* Increment a percpu refcount unless it has already been killed. Returns
	230	* %true on success; %false on failure.
	231	*
6251f997 TH	232	* Completion of percpu_ref_kill() in itself doesn't guarantee that this
	233	* function will fail. For such guarantee, percpu_ref_kill_and_confirm()
	234	* should be used. After the confirm_kill callback is invoked, it's
	235	* guaranteed that no new reference will be given out by
	236	* percpu_ref_tryget_live().
4fb6e250	237	*
6251f997	238	* This function is safe to call as long as @ref is between init and exit.
dbece3a0	239	*/
2070d50e	240	static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
dbece3a0	241	{
eecc16ba	242	unsigned long __percpu *percpu_count;
dbece3a0 TH	243	int ret = false;
dbece3a0 TH	244
a4244454	245	rcu_read_lock_sched();
dbece3a0	246
9e804d1f	247	if (__ref_is_percpu(ref, &percpu_count)) {
eecc16ba	248	this_cpu_inc(*percpu_count);
dbece3a0	249	ret = true;
6810e4a3	250	} else if (!(ref->percpu_count_ptr & __PERCPU_REF_DEAD)) {
490c79a6	251	ret = atomic_long_inc_not_zero(&ref->count);
dbece3a0 TH	252	}
dbece3a0 TH	253
a4244454	254	rcu_read_unlock_sched();
dbece3a0 TH	255
	256	return ret;
	257	}
	258
215e262f	259	/**
e8ea14cc	260	* percpu_ref_put_many - decrement a percpu refcount
ac899061	261	* @ref: percpu_ref to put
e8ea14cc	262	* @nr: number of references to put
215e262f KO	263	*
	264	* Decrement the refcount, and if 0, call the release function (which was passed
	265	* to percpu_ref_init())
6251f997 TH	266	*
6251f997 TH	267	* This function is safe to call as long as @ref is between init and exit.
215e262f	268	*/
e8ea14cc	269	static inline void percpu_ref_put_many(struct percpu_ref *ref, unsigned long nr)
215e262f	270	{
eecc16ba	271	unsigned long __percpu *percpu_count;
215e262f	272
a4244454	273	rcu_read_lock_sched();
215e262f	274
9e804d1f	275	if (__ref_is_percpu(ref, &percpu_count))
e8ea14cc JW	276	this_cpu_sub(*percpu_count, nr);
e8ea14cc JW	277	else if (unlikely(atomic_long_sub_and_test(nr, &ref->count)))
215e262f KO	278	ref->release(ref);
215e262f KO	279
a4244454	280	rcu_read_unlock_sched();
215e262f KO	281	}
215e262f KO	282
e8ea14cc JW	283	/**
	284	* percpu_ref_put - decrement a percpu refcount
	285	* @ref: percpu_ref to put
	286	*
	287	* Decrement the refcount, and if 0, call the release function (which was passed
	288	* to percpu_ref_init())
	289	*
	290	* This function is safe to call as long as @ref is between init and exit.
	291	*/
	292	static inline void percpu_ref_put(struct percpu_ref *ref)
	293	{
	294	percpu_ref_put_many(ref, 1);
	295	}
	296
4c907baf TH	297	/**
	298	* percpu_ref_is_dying - test whether a percpu refcount is dying or dead
	299	* @ref: percpu_ref to test
	300	*
	301	* Returns %true if @ref is dying or dead.
	302	*
	303	* This function is safe to call as long as @ref is between init and exit
	304	* and the caller is responsible for synchronizing against state changes.
	305	*/
	306	static inline bool percpu_ref_is_dying(struct percpu_ref *ref)
	307	{
	308	return ref->percpu_count_ptr & __PERCPU_REF_DEAD;
	309	}
	310
2d722782 TH	311	/**
	312	* percpu_ref_is_zero - test whether a percpu refcount reached zero
	313	* @ref: percpu_ref to test
	314	*
	315	* Returns %true if @ref reached zero.
6251f997 TH	316	*
6251f997 TH	317	* This function is safe to call as long as @ref is between init and exit.
2d722782 TH	318	*/
	319	static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
	320	{
eecc16ba	321	unsigned long __percpu *percpu_count;
2d722782	322
9e804d1f	323	if (__ref_is_percpu(ref, &percpu_count))
2d722782	324	return false;
e625305b	325	return !atomic_long_read(&ref->count);
2d722782 TH	326	}
2d722782 TH	327
215e262f	328	#endif