[linux-2.6-block.git] / drivers / md / dm-bio-prison-v1.c

/*
 * Copyright (C) 2012 Red Hat, Inc.
 *
 * This file is released under the GPL.
 */

#include "dm.h"
#include "dm-bio-prison-v1.h"
#include "dm-bio-prison-v2.h"

#include <linux/spinlock.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/slab.h>

/*----------------------------------------------------------------*/

#define MIN_CELLS 1024

struct dm_bio_prison {
	spinlock_t lock;
	struct rb_root cells;
	mempool_t cell_pool;
};

static struct kmem_cache *_cell_cache;

/*----------------------------------------------------------------*/

/*
 * @nr_cells should be the number of cells you want in use _concurrently_.
 * Don't confuse it with the number of distinct keys.
 */
struct dm_bio_prison *dm_bio_prison_create(void)
{
	struct dm_bio_prison *prison = kzalloc(sizeof(*prison), GFP_KERNEL);
	int ret;

	if (!prison)
		return NULL;

	spin_lock_init(&prison->lock);

	ret = mempool_init_slab_pool(&prison->cell_pool, MIN_CELLS, _cell_cache);
	if (ret) {
		kfree(prison);
		return NULL;
	}

	prison->cells = RB_ROOT;

	return prison;
}
EXPORT_SYMBOL_GPL(dm_bio_prison_create);

void dm_bio_prison_destroy(struct dm_bio_prison *prison)
{
	mempool_exit(&prison->cell_pool);
	kfree(prison);
}
EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);

struct dm_bio_prison_cell *dm_bio_prison_alloc_cell(struct dm_bio_prison *prison, gfp_t gfp)
{
	return mempool_alloc(&prison->cell_pool, gfp);
}
EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);

void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
			     struct dm_bio_prison_cell *cell)
{
	mempool_free(cell, &prison->cell_pool);
}
EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);

static void __setup_new_cell(struct dm_cell_key *key,
			     struct bio *holder,
			     struct dm_bio_prison_cell *cell)
{
       memcpy(&cell->key, key, sizeof(cell->key));
       cell->holder = holder;
       bio_list_init(&cell->bios);
}

static int cmp_keys(struct dm_cell_key *lhs,
		    struct dm_cell_key *rhs)
{
	if (lhs->virtual < rhs->virtual)
		return -1;

	if (lhs->virtual > rhs->virtual)
		return 1;

	if (lhs->dev < rhs->dev)
		return -1;

	if (lhs->dev > rhs->dev)
		return 1;

	if (lhs->block_end <= rhs->block_begin)
		return -1;

	if (lhs->block_begin >= rhs->block_end)
		return 1;

	return 0;
}

static int __bio_detain(struct dm_bio_prison *prison,
			struct dm_cell_key *key,
			struct bio *inmate,
			struct dm_bio_prison_cell *cell_prealloc,
			struct dm_bio_prison_cell **cell_result)
{
	int r;
	struct rb_node **new = &prison->cells.rb_node, *parent = NULL;

	while (*new) {
		struct dm_bio_prison_cell *cell =
			rb_entry(*new, struct dm_bio_prison_cell, node);

		r = cmp_keys(key, &cell->key);

		parent = *new;
		if (r < 0)
			new = &((*new)->rb_left);
		else if (r > 0)
			new = &((*new)->rb_right);
		else {
			if (inmate)
				bio_list_add(&cell->bios, inmate);
			*cell_result = cell;
			return 1;
		}
	}

	__setup_new_cell(key, inmate, cell_prealloc);
	*cell_result = cell_prealloc;

	rb_link_node(&cell_prealloc->node, parent, new);
	rb_insert_color(&cell_prealloc->node, &prison->cells);

	return 0;
}

static int bio_detain(struct dm_bio_prison *prison,
		      struct dm_cell_key *key,
		      struct bio *inmate,
		      struct dm_bio_prison_cell *cell_prealloc,
		      struct dm_bio_prison_cell **cell_result)
{
	int r;
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
	spin_unlock_irqrestore(&prison->lock, flags);

	return r;
}

int dm_bio_detain(struct dm_bio_prison *prison,
		  struct dm_cell_key *key,
		  struct bio *inmate,
		  struct dm_bio_prison_cell *cell_prealloc,
		  struct dm_bio_prison_cell **cell_result)
{
	return bio_detain(prison, key, inmate, cell_prealloc, cell_result);
}
EXPORT_SYMBOL_GPL(dm_bio_detain);

int dm_get_cell(struct dm_bio_prison *prison,
		struct dm_cell_key *key,
		struct dm_bio_prison_cell *cell_prealloc,
		struct dm_bio_prison_cell **cell_result)
{
	return bio_detain(prison, key, NULL, cell_prealloc, cell_result);
}
EXPORT_SYMBOL_GPL(dm_get_cell);

/*
 * @inmates must have been initialised prior to this call
 */
static void __cell_release(struct dm_bio_prison *prison,
			   struct dm_bio_prison_cell *cell,
			   struct bio_list *inmates)
{
	rb_erase(&cell->node, &prison->cells);

	if (inmates) {
		if (cell->holder)
			bio_list_add(inmates, cell->holder);
		bio_list_merge(inmates, &cell->bios);
	}
}

void dm_cell_release(struct dm_bio_prison *prison,
		     struct dm_bio_prison_cell *cell,
		     struct bio_list *bios)
{
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	__cell_release(prison, cell, bios);
	spin_unlock_irqrestore(&prison->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_cell_release);

/*
 * Sometimes we don't want the holder, just the additional bios.
 */
static void __cell_release_no_holder(struct dm_bio_prison *prison,
				     struct dm_bio_prison_cell *cell,
				     struct bio_list *inmates)
{
	rb_erase(&cell->node, &prison->cells);
	bio_list_merge(inmates, &cell->bios);
}

void dm_cell_release_no_holder(struct dm_bio_prison *prison,
			       struct dm_bio_prison_cell *cell,
			       struct bio_list *inmates)
{
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	__cell_release_no_holder(prison, cell, inmates);
	spin_unlock_irqrestore(&prison->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);

void dm_cell_error(struct dm_bio_prison *prison,
		   struct dm_bio_prison_cell *cell, blk_status_t error)
{
	struct bio_list bios;
	struct bio *bio;

	bio_list_init(&bios);
	dm_cell_release(prison, cell, &bios);

	while ((bio = bio_list_pop(&bios))) {
		bio->bi_status = error;
		bio_endio(bio);
	}
}
EXPORT_SYMBOL_GPL(dm_cell_error);

void dm_cell_visit_release(struct dm_bio_prison *prison,
			   void (*visit_fn)(void *, struct dm_bio_prison_cell *),
			   void *context,
			   struct dm_bio_prison_cell *cell)
{
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	visit_fn(context, cell);
	rb_erase(&cell->node, &prison->cells);
	spin_unlock_irqrestore(&prison->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_cell_visit_release);

static int __promote_or_release(struct dm_bio_prison *prison,
				struct dm_bio_prison_cell *cell)
{
	if (bio_list_empty(&cell->bios)) {
		rb_erase(&cell->node, &prison->cells);
		return 1;
	}

	cell->holder = bio_list_pop(&cell->bios);
	return 0;
}

int dm_cell_promote_or_release(struct dm_bio_prison *prison,
			       struct dm_bio_prison_cell *cell)
{
	int r;
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	r = __promote_or_release(prison, cell);
	spin_unlock_irqrestore(&prison->lock, flags);

	return r;
}
EXPORT_SYMBOL_GPL(dm_cell_promote_or_release);

/*----------------------------------------------------------------*/

#define DEFERRED_SET_SIZE 64

struct dm_deferred_entry {
	struct dm_deferred_set *ds;
	unsigned count;
	struct list_head work_items;
};

struct dm_deferred_set {
	spinlock_t lock;
	unsigned current_entry;
	unsigned sweeper;
	struct dm_deferred_entry entries[DEFERRED_SET_SIZE];
};

struct dm_deferred_set *dm_deferred_set_create(void)
{
	int i;
	struct dm_deferred_set *ds;

	ds = kmalloc(sizeof(*ds), GFP_KERNEL);
	if (!ds)
		return NULL;

	spin_lock_init(&ds->lock);
	ds->current_entry = 0;
	ds->sweeper = 0;
	for (i = 0; i < DEFERRED_SET_SIZE; i++) {
		ds->entries[i].ds = ds;
		ds->entries[i].count = 0;
		INIT_LIST_HEAD(&ds->entries[i].work_items);
	}

	return ds;
}
EXPORT_SYMBOL_GPL(dm_deferred_set_create);

void dm_deferred_set_destroy(struct dm_deferred_set *ds)
{
	kfree(ds);
}
EXPORT_SYMBOL_GPL(dm_deferred_set_destroy);

struct dm_deferred_entry *dm_deferred_entry_inc(struct dm_deferred_set *ds)
{
	unsigned long flags;
	struct dm_deferred_entry *entry;

	spin_lock_irqsave(&ds->lock, flags);
	entry = ds->entries + ds->current_entry;
	entry->count++;
	spin_unlock_irqrestore(&ds->lock, flags);

	return entry;
}
EXPORT_SYMBOL_GPL(dm_deferred_entry_inc);

static unsigned ds_next(unsigned index)
{
	return (index + 1) % DEFERRED_SET_SIZE;
}

static void __sweep(struct dm_deferred_set *ds, struct list_head *head)
{
	while ((ds->sweeper != ds->current_entry) &&
	       !ds->entries[ds->sweeper].count) {
		list_splice_init(&ds->entries[ds->sweeper].work_items, head);
		ds->sweeper = ds_next(ds->sweeper);
	}

	if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
		list_splice_init(&ds->entries[ds->sweeper].work_items, head);
}

void dm_deferred_entry_dec(struct dm_deferred_entry *entry, struct list_head *head)
{
	unsigned long flags;

	spin_lock_irqsave(&entry->ds->lock, flags);
	BUG_ON(!entry->count);
	--entry->count;
	__sweep(entry->ds, head);
	spin_unlock_irqrestore(&entry->ds->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_deferred_entry_dec);

/*
 * Returns 1 if deferred or 0 if no pending items to delay job.
 */
int dm_deferred_set_add_work(struct dm_deferred_set *ds, struct list_head *work)
{
	int r = 1;
	unsigned long flags;
	unsigned next_entry;

	spin_lock_irqsave(&ds->lock, flags);
	if ((ds->sweeper == ds->current_entry) &&
	    !ds->entries[ds->current_entry].count)
		r = 0;
	else {
		list_add(work, &ds->entries[ds->current_entry].work_items);
		next_entry = ds_next(ds->current_entry);
		if (!ds->entries[next_entry].count)
			ds->current_entry = next_entry;
	}
	spin_unlock_irqrestore(&ds->lock, flags);

	return r;
}
EXPORT_SYMBOL_GPL(dm_deferred_set_add_work);

/*----------------------------------------------------------------*/

static int __init dm_bio_prison_init_v1(void)
{
	_cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
	if (!_cell_cache)
		return -ENOMEM;

	return 0;
}

static void dm_bio_prison_exit_v1(void)
{
	kmem_cache_destroy(_cell_cache);
	_cell_cache = NULL;
}

static int (*_inits[])(void) __initdata = {
	dm_bio_prison_init_v1,
	dm_bio_prison_init_v2,
};

static void (*_exits[])(void) = {
	dm_bio_prison_exit_v1,
	dm_bio_prison_exit_v2,
};

static int __init dm_bio_prison_init(void)
{
	const int count = ARRAY_SIZE(_inits);

	int r, i;

	for (i = 0; i < count; i++) {
		r = _inits[i]();
		if (r)
			goto bad;
	}

	return 0;

      bad:
	while (i--)
		_exits[i]();

	return r;
}

static void __exit dm_bio_prison_exit(void)
{
	int i = ARRAY_SIZE(_exits);

	while (i--)
		_exits[i]();
}

/*
 * module hooks
 */
module_init(dm_bio_prison_init);
module_exit(dm_bio_prison_exit);

MODULE_DESCRIPTION(DM_NAME " bio prison");
MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
MODULE_LICENSE("GPL");
Commit	Line	Data
4f81a417 MS	1	/*
	2	* Copyright (C) 2012 Red Hat, Inc.
	3	*
	4	* This file is released under the GPL.
	5	*/
	6
	7	#include "dm.h"
742c8fdc JT	8	#include "dm-bio-prison-v1.h"
742c8fdc JT	9	#include "dm-bio-prison-v2.h"
4f81a417 MS	10
	11	#include <linux/spinlock.h>
	12	#include <linux/mempool.h>
	13	#include <linux/module.h>
	14	#include <linux/slab.h>
	15
	16	/----------------------------------------------------------------/
	17
a195db2d	18	#define MIN_CELLS 1024
adcc4447 HM	19
adcc4447 HM	20	struct dm_bio_prison {
a195db2d	21	spinlock_t lock;
a195db2d	22	struct rb_root cells;
72d711c8	23	mempool_t cell_pool;
4f81a417 MS	24	};
4f81a417 MS	25
4f81a417 MS	26	static struct kmem_cache *_cell_cache;
4f81a417 MS	27
a195db2d	28	/----------------------------------------------------------------/
adcc4447	29
4f81a417 MS	30	/*
	31	* @nr_cells should be the number of cells you want in use _concurrently_.
	32	* Don't confuse it with the number of distinct keys.
	33	*/
a195db2d	34	struct dm_bio_prison *dm_bio_prison_create(void)
4f81a417	35	{
d3775354	36	struct dm_bio_prison prison = kzalloc(sizeof(prison), GFP_KERNEL);
6f1c819c	37	int ret;
4f81a417 MS	38
	39	if (!prison)
	40	return NULL;
	41
a195db2d JT	42	spin_lock_init(&prison->lock);
a195db2d JT	43
6f1c819c KO	44	ret = mempool_init_slab_pool(&prison->cell_pool, MIN_CELLS, _cell_cache);
6f1c819c KO	45	if (ret) {
4f81a417 MS	46	kfree(prison);
	47	return NULL;
	48	}
	49
a195db2d	50	prison->cells = RB_ROOT;
4f81a417 MS	51
	52	return prison;
	53	}
	54	EXPORT_SYMBOL_GPL(dm_bio_prison_create);
	55
	56	void dm_bio_prison_destroy(struct dm_bio_prison *prison)
	57	{
6f1c819c	58	mempool_exit(&prison->cell_pool);
4f81a417 MS	59	kfree(prison);
	60	}
	61	EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);
	62
6beca5eb JT	63	struct dm_bio_prison_cell dm_bio_prison_alloc_cell(struct dm_bio_prison prison, gfp_t gfp)
6beca5eb JT	64	{
6f1c819c	65	return mempool_alloc(&prison->cell_pool, gfp);
6beca5eb JT	66	}
	67	EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);
	68
	69	void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
	70	struct dm_bio_prison_cell *cell)
	71	{
6f1c819c	72	mempool_free(cell, &prison->cell_pool);
6beca5eb JT	73	}
	74	EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);
	75
a195db2d JT	76	static void __setup_new_cell(struct dm_cell_key *key,
	77	struct bio *holder,
	78	struct dm_bio_prison_cell *cell)
4f81a417	79	{
a195db2d JT	80	memcpy(&cell->key, key, sizeof(cell->key));
	81	cell->holder = holder;
	82	bio_list_init(&cell->bios);
4f81a417 MS	83	}
4f81a417 MS	84
a195db2d JT	85	static int cmp_keys(struct dm_cell_key *lhs,
a195db2d JT	86	struct dm_cell_key *rhs)
4f81a417	87	{
a195db2d JT	88	if (lhs->virtual < rhs->virtual)
a195db2d JT	89	return -1;
4f81a417	90
a195db2d JT	91	if (lhs->virtual > rhs->virtual)
a195db2d JT	92	return 1;
adcc4447	93
a195db2d JT	94	if (lhs->dev < rhs->dev)
a195db2d JT	95	return -1;
4f81a417	96
a195db2d JT	97	if (lhs->dev > rhs->dev)
a195db2d JT	98	return 1;
4f81a417	99
5f274d88	100	if (lhs->block_end <= rhs->block_begin)
a195db2d	101	return -1;
4f81a417	102
5f274d88	103	if (lhs->block_begin >= rhs->block_end)
a195db2d JT	104	return 1;
	105
	106	return 0;
6beca5eb	107	}
4f81a417	108
a195db2d	109	static int __bio_detain(struct dm_bio_prison *prison,
6beca5eb JT	110	struct dm_cell_key *key,
	111	struct bio *inmate,
	112	struct dm_bio_prison_cell *cell_prealloc,
	113	struct dm_bio_prison_cell **cell_result)
	114	{
a195db2d JT	115	int r;
	116	struct rb_node *new = &prison->cells.rb_node, parent = NULL;
	117
	118	while (*new) {
	119	struct dm_bio_prison_cell *cell =
6e333d0b	120	rb_entry(*new, struct dm_bio_prison_cell, node);
a195db2d JT	121
	122	r = cmp_keys(key, &cell->key);
	123
	124	parent = *new;
	125	if (r < 0)
	126	new = &((*new)->rb_left);
	127	else if (r > 0)
	128	new = &((*new)->rb_right);
	129	else {
	130	if (inmate)
	131	bio_list_add(&cell->bios, inmate);
	132	*cell_result = cell;
	133	return 1;
	134	}
4f81a417 MS	135	}
4f81a417 MS	136
a195db2d	137	__setup_new_cell(key, inmate, cell_prealloc);
6beca5eb	138	*cell_result = cell_prealloc;
a195db2d JT	139
	140	rb_link_node(&cell_prealloc->node, parent, new);
	141	rb_insert_color(&cell_prealloc->node, &prison->cells);
	142
6beca5eb JT	143	return 0;
6beca5eb JT	144	}
4f81a417	145
6beca5eb JT	146	static int bio_detain(struct dm_bio_prison *prison,
	147	struct dm_cell_key *key,
	148	struct bio *inmate,
	149	struct dm_bio_prison_cell *cell_prealloc,
	150	struct dm_bio_prison_cell **cell_result)
	151	{
	152	int r;
	153	unsigned long flags;
4f81a417	154
a195db2d JT	155	spin_lock_irqsave(&prison->lock, flags);
	156	r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
	157	spin_unlock_irqrestore(&prison->lock, flags);
4f81a417	158
4f81a417 MS	159	return r;
4f81a417 MS	160	}
6beca5eb JT	161
	162	int dm_bio_detain(struct dm_bio_prison *prison,
	163	struct dm_cell_key *key,
	164	struct bio *inmate,
	165	struct dm_bio_prison_cell *cell_prealloc,
	166	struct dm_bio_prison_cell **cell_result)
	167	{
	168	return bio_detain(prison, key, inmate, cell_prealloc, cell_result);
	169	}
4f81a417 MS	170	EXPORT_SYMBOL_GPL(dm_bio_detain);
4f81a417 MS	171
c6b4fcba JT	172	int dm_get_cell(struct dm_bio_prison *prison,
	173	struct dm_cell_key *key,
	174	struct dm_bio_prison_cell *cell_prealloc,
	175	struct dm_bio_prison_cell **cell_result)
	176	{
	177	return bio_detain(prison, key, NULL, cell_prealloc, cell_result);
	178	}
	179	EXPORT_SYMBOL_GPL(dm_get_cell);
	180
4f81a417 MS	181	/*
	182	* @inmates must have been initialised prior to this call
	183	*/
a195db2d JT	184	static void __cell_release(struct dm_bio_prison *prison,
a195db2d JT	185	struct dm_bio_prison_cell *cell,
6beca5eb	186	struct bio_list *inmates)
4f81a417	187	{
a195db2d	188	rb_erase(&cell->node, &prison->cells);
4f81a417 MS	189
4f81a417 MS	190	if (inmates) {
6beca5eb JT	191	if (cell->holder)
6beca5eb JT	192	bio_list_add(inmates, cell->holder);
4f81a417 MS	193	bio_list_merge(inmates, &cell->bios);
4f81a417 MS	194	}
4f81a417 MS	195	}
4f81a417 MS	196
6beca5eb JT	197	void dm_cell_release(struct dm_bio_prison *prison,
	198	struct dm_bio_prison_cell *cell,
	199	struct bio_list *bios)
4f81a417 MS	200	{
4f81a417 MS	201	unsigned long flags;
4f81a417	202
a195db2d JT	203	spin_lock_irqsave(&prison->lock, flags);
	204	__cell_release(prison, cell, bios);
	205	spin_unlock_irqrestore(&prison->lock, flags);
4f81a417 MS	206	}
	207	EXPORT_SYMBOL_GPL(dm_cell_release);
	208
4f81a417 MS	209	/*
	210	* Sometimes we don't want the holder, just the additional bios.
	211	*/
a195db2d JT	212	static void __cell_release_no_holder(struct dm_bio_prison *prison,
a195db2d JT	213	struct dm_bio_prison_cell *cell,
6beca5eb	214	struct bio_list *inmates)
4f81a417	215	{
a195db2d	216	rb_erase(&cell->node, &prison->cells);
4f81a417	217	bio_list_merge(inmates, &cell->bios);
4f81a417 MS	218	}
4f81a417 MS	219
6beca5eb JT	220	void dm_cell_release_no_holder(struct dm_bio_prison *prison,
	221	struct dm_bio_prison_cell *cell,
	222	struct bio_list *inmates)
4f81a417 MS	223	{
4f81a417 MS	224	unsigned long flags;
4f81a417	225
a195db2d JT	226	spin_lock_irqsave(&prison->lock, flags);
	227	__cell_release_no_holder(prison, cell, inmates);
	228	spin_unlock_irqrestore(&prison->lock, flags);
4f81a417 MS	229	}
	230	EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);
	231
6beca5eb	232	void dm_cell_error(struct dm_bio_prison *prison,
4e4cbee9	233	struct dm_bio_prison_cell *cell, blk_status_t error)
4f81a417	234	{
4f81a417 MS	235	struct bio_list bios;
4f81a417 MS	236	struct bio *bio;
4f81a417 MS	237
4f81a417 MS	238	bio_list_init(&bios);
adcc4447	239	dm_cell_release(prison, cell, &bios);
4f81a417	240
4246a0b6	241	while ((bio = bio_list_pop(&bios))) {
4e4cbee9	242	bio->bi_status = error;
4246a0b6 CH	243	bio_endio(bio);
4246a0b6 CH	244	}
4f81a417 MS	245	}
	246	EXPORT_SYMBOL_GPL(dm_cell_error);
	247
2d759a46 JT	248	void dm_cell_visit_release(struct dm_bio_prison *prison,
	249	void (visit_fn)(void , struct dm_bio_prison_cell *),
	250	void *context,
	251	struct dm_bio_prison_cell *cell)
	252	{
	253	unsigned long flags;
	254
	255	spin_lock_irqsave(&prison->lock, flags);
	256	visit_fn(context, cell);
	257	rb_erase(&cell->node, &prison->cells);
	258	spin_unlock_irqrestore(&prison->lock, flags);
	259	}
	260	EXPORT_SYMBOL_GPL(dm_cell_visit_release);
	261
3cdf93f9 JT	262	static int __promote_or_release(struct dm_bio_prison *prison,
	263	struct dm_bio_prison_cell *cell)
	264	{
	265	if (bio_list_empty(&cell->bios)) {
	266	rb_erase(&cell->node, &prison->cells);
	267	return 1;
	268	}
	269
	270	cell->holder = bio_list_pop(&cell->bios);
	271	return 0;
	272	}
	273
	274	int dm_cell_promote_or_release(struct dm_bio_prison *prison,
	275	struct dm_bio_prison_cell *cell)
	276	{
	277	int r;
	278	unsigned long flags;
	279
	280	spin_lock_irqsave(&prison->lock, flags);
	281	r = __promote_or_release(prison, cell);
	282	spin_unlock_irqrestore(&prison->lock, flags);
	283
	284	return r;
	285	}
	286	EXPORT_SYMBOL_GPL(dm_cell_promote_or_release);
	287
4f81a417 MS	288	/----------------------------------------------------------------/
	289
	290	#define DEFERRED_SET_SIZE 64
	291
	292	struct dm_deferred_entry {
	293	struct dm_deferred_set *ds;
	294	unsigned count;
	295	struct list_head work_items;
	296	};
	297
	298	struct dm_deferred_set {
	299	spinlock_t lock;
	300	unsigned current_entry;
	301	unsigned sweeper;
	302	struct dm_deferred_entry entries[DEFERRED_SET_SIZE];
	303	};
	304
	305	struct dm_deferred_set *dm_deferred_set_create(void)
	306	{
	307	int i;
	308	struct dm_deferred_set *ds;
	309
	310	ds = kmalloc(sizeof(*ds), GFP_KERNEL);
	311	if (!ds)
	312	return NULL;
	313
	314	spin_lock_init(&ds->lock);
	315	ds->current_entry = 0;
	316	ds->sweeper = 0;
	317	for (i = 0; i < DEFERRED_SET_SIZE; i++) {
	318	ds->entries[i].ds = ds;
	319	ds->entries[i].count = 0;
	320	INIT_LIST_HEAD(&ds->entries[i].work_items);
	321	}
	322
	323	return ds;
	324	}
	325	EXPORT_SYMBOL_GPL(dm_deferred_set_create);
	326
	327	void dm_deferred_set_destroy(struct dm_deferred_set *ds)
	328	{
	329	kfree(ds);
	330	}
	331	EXPORT_SYMBOL_GPL(dm_deferred_set_destroy);
	332
	333	struct dm_deferred_entry dm_deferred_entry_inc(struct dm_deferred_set ds)
	334	{
	335	unsigned long flags;
	336	struct dm_deferred_entry *entry;
	337
	338	spin_lock_irqsave(&ds->lock, flags);
	339	entry = ds->entries + ds->current_entry;
	340	entry->count++;
	341	spin_unlock_irqrestore(&ds->lock, flags);
	342
	343	return entry;
	344	}
	345	EXPORT_SYMBOL_GPL(dm_deferred_entry_inc);
	346
	347	static unsigned ds_next(unsigned index)
	348	{
	349	return (index + 1) % DEFERRED_SET_SIZE;
	350	}
	351
352	static void __sweep(struct dm_deferred_set ds, struct list_head head)
353	{
354	while ((ds->sweeper != ds->current_entry) &&
355	!ds->entries[ds->sweeper].count) {
356	list_splice_init(&ds->entries[ds->sweeper].work_items, head);
357	ds->sweeper = ds_next(ds->sweeper);
358	}
359
360	if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
361	list_splice_init(&ds->entries[ds->sweeper].work_items, head);
362	}
363
364	void dm_deferred_entry_dec(struct dm_deferred_entry entry, struct list_head head)
365	{
366	unsigned long flags;
367
368	spin_lock_irqsave(&entry->ds->lock, flags);
369	BUG_ON(!entry->count);
370	--entry->count;
371	__sweep(entry->ds, head);
372	spin_unlock_irqrestore(&entry->ds->lock, flags);
373	}
374	EXPORT_SYMBOL_GPL(dm_deferred_entry_dec);
375
376	/*
377	* Returns 1 if deferred or 0 if no pending items to delay job.
378	*/
379	int dm_deferred_set_add_work(struct dm_deferred_set ds, struct list_head work)
380	{
381	int r = 1;
382	unsigned long flags;
383	unsigned next_entry;
384
385	spin_lock_irqsave(&ds->lock, flags);
386	if ((ds->sweeper == ds->current_entry) &&
387	!ds->entries[ds->current_entry].count)
388	r = 0;
389	else {
390	list_add(work, &ds->entries[ds->current_entry].work_items);
391	next_entry = ds_next(ds->current_entry);
392	if (!ds->entries[next_entry].count)
393	ds->current_entry = next_entry;
394	}
395	spin_unlock_irqrestore(&ds->lock, flags);
396
397	return r;
398	}
399	EXPORT_SYMBOL_GPL(dm_deferred_set_add_work);
400
401	/----------------------------------------------------------------/
402
742c8fdc	403	static int __init dm_bio_prison_init_v1(void)
4f81a417 MS	404	{
	405	_cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
	406	if (!_cell_cache)
	407	return -ENOMEM;
	408
	409	return 0;
	410	}
	411
742c8fdc	412	static void dm_bio_prison_exit_v1(void)
4f81a417 MS	413	{
	414	kmem_cache_destroy(_cell_cache);
	415	_cell_cache = NULL;
	416	}
	417
742c8fdc JT	418	static int (*_inits[])(void) __initdata = {
	419	dm_bio_prison_init_v1,
	420	dm_bio_prison_init_v2,
	421	};
	422
	423	static void (*_exits[])(void) = {
	424	dm_bio_prison_exit_v1,
	425	dm_bio_prison_exit_v2,
	426	};
	427
	428	static int __init dm_bio_prison_init(void)
	429	{
	430	const int count = ARRAY_SIZE(_inits);
	431
	432	int r, i;
	433
	434	for (i = 0; i < count; i++) {
	435	r = _inits[i]();
	436	if (r)
	437	goto bad;
	438	}
	439
	440	return 0;
	441
	442	bad:
	443	while (i--)
	444	_exits[i]();
	445
	446	return r;
	447	}
	448
	449	static void __exit dm_bio_prison_exit(void)
	450	{
	451	int i = ARRAY_SIZE(_exits);
	452
	453	while (i--)
	454	_exits[i]();
	455	}
	456
4f81a417 MS	457	/*
	458	* module hooks
	459	*/
	460	module_init(dm_bio_prison_init);
	461	module_exit(dm_bio_prison_exit);
	462
	463	MODULE_DESCRIPTION(DM_NAME " bio prison");
	464	MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
	465	MODULE_LICENSE("GPL");