[linux-2.6-block.git] / drivers / md / dm-bio-prison.c

/*
 * Copyright (C) 2012 Red Hat, Inc.
 *
 * This file is released under the GPL.
 */

#include "dm.h"
#include "dm-bio-prison.h"

#include <linux/spinlock.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/slab.h>

/*----------------------------------------------------------------*/

#define MIN_CELLS 1024

struct dm_bio_prison {
	spinlock_t lock;
	mempool_t *cell_pool;
	struct rb_root cells;
};

static struct kmem_cache *_cell_cache;

/*----------------------------------------------------------------*/

/*
 * @nr_cells should be the number of cells you want in use _concurrently_.
 * Don't confuse it with the number of distinct keys.
 */
struct dm_bio_prison *dm_bio_prison_create(void)
{
	struct dm_bio_prison *prison = kmalloc(sizeof(*prison), GFP_KERNEL);

	if (!prison)
		return NULL;

	spin_lock_init(&prison->lock);

	prison->cell_pool = mempool_create_slab_pool(MIN_CELLS, _cell_cache);
	if (!prison->cell_pool) {
		kfree(prison);
		return NULL;
	}

	prison->cells = RB_ROOT;

	return prison;
}
EXPORT_SYMBOL_GPL(dm_bio_prison_create);

void dm_bio_prison_destroy(struct dm_bio_prison *prison)
{
	mempool_destroy(prison->cell_pool);
	kfree(prison);
}
EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);

struct dm_bio_prison_cell *dm_bio_prison_alloc_cell(struct dm_bio_prison *prison, gfp_t gfp)
{
	return mempool_alloc(prison->cell_pool, gfp);
}
EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);

void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
			     struct dm_bio_prison_cell *cell)
{
	mempool_free(cell, prison->cell_pool);
}
EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);

static void __setup_new_cell(struct dm_cell_key *key,
			     struct bio *holder,
			     struct dm_bio_prison_cell *cell)
{
       memcpy(&cell->key, key, sizeof(cell->key));
       cell->holder = holder;
       bio_list_init(&cell->bios);
}

static int cmp_keys(struct dm_cell_key *lhs,
		    struct dm_cell_key *rhs)
{
	if (lhs->virtual < rhs->virtual)
		return -1;

	if (lhs->virtual > rhs->virtual)
		return 1;

	if (lhs->dev < rhs->dev)
		return -1;

	if (lhs->dev > rhs->dev)
		return 1;

	if (lhs->block_end <= rhs->block_begin)
		return -1;

	if (lhs->block_begin >= rhs->block_end)
		return 1;

	return 0;
}

static int __bio_detain(struct dm_bio_prison *prison,
			struct dm_cell_key *key,
			struct bio *inmate,
			struct dm_bio_prison_cell *cell_prealloc,
			struct dm_bio_prison_cell **cell_result)
{
	int r;
	struct rb_node **new = &prison->cells.rb_node, *parent = NULL;

	while (*new) {
		struct dm_bio_prison_cell *cell =
			container_of(*new, struct dm_bio_prison_cell, node);

		r = cmp_keys(key, &cell->key);

		parent = *new;
		if (r < 0)
			new = &((*new)->rb_left);
		else if (r > 0)
			new = &((*new)->rb_right);
		else {
			if (inmate)
				bio_list_add(&cell->bios, inmate);
			*cell_result = cell;
			return 1;
		}
	}

	__setup_new_cell(key, inmate, cell_prealloc);
	*cell_result = cell_prealloc;

	rb_link_node(&cell_prealloc->node, parent, new);
	rb_insert_color(&cell_prealloc->node, &prison->cells);

	return 0;
}

static int bio_detain(struct dm_bio_prison *prison,
		      struct dm_cell_key *key,
		      struct bio *inmate,
		      struct dm_bio_prison_cell *cell_prealloc,
		      struct dm_bio_prison_cell **cell_result)
{
	int r;
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
	spin_unlock_irqrestore(&prison->lock, flags);

	return r;
}

int dm_bio_detain(struct dm_bio_prison *prison,
		  struct dm_cell_key *key,
		  struct bio *inmate,
		  struct dm_bio_prison_cell *cell_prealloc,
		  struct dm_bio_prison_cell **cell_result)
{
	return bio_detain(prison, key, inmate, cell_prealloc, cell_result);
}
EXPORT_SYMBOL_GPL(dm_bio_detain);

int dm_get_cell(struct dm_bio_prison *prison,
		struct dm_cell_key *key,
		struct dm_bio_prison_cell *cell_prealloc,
		struct dm_bio_prison_cell **cell_result)
{
	return bio_detain(prison, key, NULL, cell_prealloc, cell_result);
}
EXPORT_SYMBOL_GPL(dm_get_cell);

/*
 * @inmates must have been initialised prior to this call
 */
static void __cell_release(struct dm_bio_prison *prison,
			   struct dm_bio_prison_cell *cell,
			   struct bio_list *inmates)
{
	rb_erase(&cell->node, &prison->cells);

	if (inmates) {
		if (cell->holder)
			bio_list_add(inmates, cell->holder);
		bio_list_merge(inmates, &cell->bios);
	}
}

void dm_cell_release(struct dm_bio_prison *prison,
		     struct dm_bio_prison_cell *cell,
		     struct bio_list *bios)
{
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	__cell_release(prison, cell, bios);
	spin_unlock_irqrestore(&prison->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_cell_release);

/*
 * Sometimes we don't want the holder, just the additional bios.
 */
static void __cell_release_no_holder(struct dm_bio_prison *prison,
				     struct dm_bio_prison_cell *cell,
				     struct bio_list *inmates)
{
	rb_erase(&cell->node, &prison->cells);
	bio_list_merge(inmates, &cell->bios);
}

void dm_cell_release_no_holder(struct dm_bio_prison *prison,
			       struct dm_bio_prison_cell *cell,
			       struct bio_list *inmates)
{
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	__cell_release_no_holder(prison, cell, inmates);
	spin_unlock_irqrestore(&prison->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);

void dm_cell_error(struct dm_bio_prison *prison,
		   struct dm_bio_prison_cell *cell, int error)
{
	struct bio_list bios;
	struct bio *bio;

	bio_list_init(&bios);
	dm_cell_release(prison, cell, &bios);

	while ((bio = bio_list_pop(&bios)))
		bio_endio(bio, error);
}
EXPORT_SYMBOL_GPL(dm_cell_error);

void dm_cell_visit_release(struct dm_bio_prison *prison,
			   void (*visit_fn)(void *, struct dm_bio_prison_cell *),
			   void *context,
			   struct dm_bio_prison_cell *cell)
{
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	visit_fn(context, cell);
	rb_erase(&cell->node, &prison->cells);
	spin_unlock_irqrestore(&prison->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_cell_visit_release);

/*----------------------------------------------------------------*/

#define DEFERRED_SET_SIZE 64

struct dm_deferred_entry {
	struct dm_deferred_set *ds;
	unsigned count;
	struct list_head work_items;
};

struct dm_deferred_set {
	spinlock_t lock;
	unsigned current_entry;
	unsigned sweeper;
	struct dm_deferred_entry entries[DEFERRED_SET_SIZE];
};

struct dm_deferred_set *dm_deferred_set_create(void)
{
	int i;
	struct dm_deferred_set *ds;

	ds = kmalloc(sizeof(*ds), GFP_KERNEL);
	if (!ds)
		return NULL;

	spin_lock_init(&ds->lock);
	ds->current_entry = 0;
	ds->sweeper = 0;
	for (i = 0; i < DEFERRED_SET_SIZE; i++) {
		ds->entries[i].ds = ds;
		ds->entries[i].count = 0;
		INIT_LIST_HEAD(&ds->entries[i].work_items);
	}

	return ds;
}
EXPORT_SYMBOL_GPL(dm_deferred_set_create);

void dm_deferred_set_destroy(struct dm_deferred_set *ds)
{
	kfree(ds);
}
EXPORT_SYMBOL_GPL(dm_deferred_set_destroy);

struct dm_deferred_entry *dm_deferred_entry_inc(struct dm_deferred_set *ds)
{
	unsigned long flags;
	struct dm_deferred_entry *entry;

	spin_lock_irqsave(&ds->lock, flags);
	entry = ds->entries + ds->current_entry;
	entry->count++;
	spin_unlock_irqrestore(&ds->lock, flags);

	return entry;
}
EXPORT_SYMBOL_GPL(dm_deferred_entry_inc);

static unsigned ds_next(unsigned index)
{
	return (index + 1) % DEFERRED_SET_SIZE;
}

static void __sweep(struct dm_deferred_set *ds, struct list_head *head)
{
	while ((ds->sweeper != ds->current_entry) &&
	       !ds->entries[ds->sweeper].count) {
		list_splice_init(&ds->entries[ds->sweeper].work_items, head);
		ds->sweeper = ds_next(ds->sweeper);
	}

	if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
		list_splice_init(&ds->entries[ds->sweeper].work_items, head);
}

void dm_deferred_entry_dec(struct dm_deferred_entry *entry, struct list_head *head)
{
	unsigned long flags;

	spin_lock_irqsave(&entry->ds->lock, flags);
	BUG_ON(!entry->count);
	--entry->count;
	__sweep(entry->ds, head);
	spin_unlock_irqrestore(&entry->ds->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_deferred_entry_dec);

/*
 * Returns 1 if deferred or 0 if no pending items to delay job.
 */
int dm_deferred_set_add_work(struct dm_deferred_set *ds, struct list_head *work)
{
	int r = 1;
	unsigned long flags;
	unsigned next_entry;

	spin_lock_irqsave(&ds->lock, flags);
	if ((ds->sweeper == ds->current_entry) &&
	    !ds->entries[ds->current_entry].count)
		r = 0;
	else {
		list_add(work, &ds->entries[ds->current_entry].work_items);
		next_entry = ds_next(ds->current_entry);
		if (!ds->entries[next_entry].count)
			ds->current_entry = next_entry;
	}
	spin_unlock_irqrestore(&ds->lock, flags);

	return r;
}
EXPORT_SYMBOL_GPL(dm_deferred_set_add_work);

/*----------------------------------------------------------------*/

static int __init dm_bio_prison_init(void)
{
	_cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
	if (!_cell_cache)
		return -ENOMEM;

	return 0;
}

static void __exit dm_bio_prison_exit(void)
{
	kmem_cache_destroy(_cell_cache);
	_cell_cache = NULL;
}

/*
 * module hooks
 */
module_init(dm_bio_prison_init);
module_exit(dm_bio_prison_exit);

MODULE_DESCRIPTION(DM_NAME " bio prison");
MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
MODULE_LICENSE("GPL");
Commit	Line	Data
4f81a417 MS	1	/*
	2	* Copyright (C) 2012 Red Hat, Inc.
	3	*
	4	* This file is released under the GPL.
	5	*/
	6
	7	#include "dm.h"
	8	#include "dm-bio-prison.h"
	9
	10	#include <linux/spinlock.h>
	11	#include <linux/mempool.h>
	12	#include <linux/module.h>
	13	#include <linux/slab.h>
	14
	15	/----------------------------------------------------------------/
	16
a195db2d	17	#define MIN_CELLS 1024
adcc4447 HM	18
adcc4447 HM	19	struct dm_bio_prison {
a195db2d	20	spinlock_t lock;
4f81a417	21	mempool_t *cell_pool;
a195db2d	22	struct rb_root cells;
4f81a417 MS	23	};
4f81a417 MS	24
4f81a417 MS	25	static struct kmem_cache *_cell_cache;
4f81a417 MS	26
a195db2d	27	/----------------------------------------------------------------/
adcc4447	28
4f81a417 MS	29	/*
	30	* @nr_cells should be the number of cells you want in use _concurrently_.
	31	* Don't confuse it with the number of distinct keys.
	32	*/
a195db2d	33	struct dm_bio_prison *dm_bio_prison_create(void)
4f81a417	34	{
a195db2d	35	struct dm_bio_prison prison = kmalloc(sizeof(prison), GFP_KERNEL);
4f81a417 MS	36
	37	if (!prison)
	38	return NULL;
	39
a195db2d JT	40	spin_lock_init(&prison->lock);
	41
	42	prison->cell_pool = mempool_create_slab_pool(MIN_CELLS, _cell_cache);
4f81a417 MS	43	if (!prison->cell_pool) {
	44	kfree(prison);
	45	return NULL;
	46	}
	47
a195db2d	48	prison->cells = RB_ROOT;
4f81a417 MS	49
	50	return prison;
	51	}
	52	EXPORT_SYMBOL_GPL(dm_bio_prison_create);
	53
	54	void dm_bio_prison_destroy(struct dm_bio_prison *prison)
	55	{
	56	mempool_destroy(prison->cell_pool);
	57	kfree(prison);
	58	}
	59	EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);
	60
6beca5eb JT	61	struct dm_bio_prison_cell dm_bio_prison_alloc_cell(struct dm_bio_prison prison, gfp_t gfp)
	62	{
	63	return mempool_alloc(prison->cell_pool, gfp);
	64	}
	65	EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);
	66
	67	void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
	68	struct dm_bio_prison_cell *cell)
	69	{
	70	mempool_free(cell, prison->cell_pool);
	71	}
	72	EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);
	73
a195db2d JT	74	static void __setup_new_cell(struct dm_cell_key *key,
	75	struct bio *holder,
	76	struct dm_bio_prison_cell *cell)
4f81a417	77	{
a195db2d JT	78	memcpy(&cell->key, key, sizeof(cell->key));
	79	cell->holder = holder;
	80	bio_list_init(&cell->bios);
4f81a417 MS	81	}
4f81a417 MS	82
a195db2d JT	83	static int cmp_keys(struct dm_cell_key *lhs,
a195db2d JT	84	struct dm_cell_key *rhs)
4f81a417	85	{
a195db2d JT	86	if (lhs->virtual < rhs->virtual)
a195db2d JT	87	return -1;
4f81a417	88
a195db2d JT	89	if (lhs->virtual > rhs->virtual)
a195db2d JT	90	return 1;
adcc4447	91
a195db2d JT	92	if (lhs->dev < rhs->dev)
a195db2d JT	93	return -1;
4f81a417	94
a195db2d JT	95	if (lhs->dev > rhs->dev)
a195db2d JT	96	return 1;
4f81a417	97
5f274d88	98	if (lhs->block_end <= rhs->block_begin)
a195db2d	99	return -1;
4f81a417	100
5f274d88	101	if (lhs->block_begin >= rhs->block_end)
a195db2d JT	102	return 1;
	103
	104	return 0;
6beca5eb	105	}
4f81a417	106
a195db2d	107	static int __bio_detain(struct dm_bio_prison *prison,
6beca5eb JT	108	struct dm_cell_key *key,
	109	struct bio *inmate,
	110	struct dm_bio_prison_cell *cell_prealloc,
	111	struct dm_bio_prison_cell **cell_result)
	112	{
a195db2d JT	113	int r;
	114	struct rb_node *new = &prison->cells.rb_node, parent = NULL;
	115
	116	while (*new) {
	117	struct dm_bio_prison_cell *cell =
	118	container_of(*new, struct dm_bio_prison_cell, node);
	119
	120	r = cmp_keys(key, &cell->key);
	121
	122	parent = *new;
	123	if (r < 0)
	124	new = &((*new)->rb_left);
	125	else if (r > 0)
	126	new = &((*new)->rb_right);
	127	else {
	128	if (inmate)
	129	bio_list_add(&cell->bios, inmate);
	130	*cell_result = cell;
	131	return 1;
	132	}
4f81a417 MS	133	}
4f81a417 MS	134
a195db2d	135	__setup_new_cell(key, inmate, cell_prealloc);
6beca5eb	136	*cell_result = cell_prealloc;
a195db2d JT	137
	138	rb_link_node(&cell_prealloc->node, parent, new);
	139	rb_insert_color(&cell_prealloc->node, &prison->cells);
	140
6beca5eb JT	141	return 0;
6beca5eb JT	142	}
4f81a417	143
6beca5eb JT	144	static int bio_detain(struct dm_bio_prison *prison,
	145	struct dm_cell_key *key,
	146	struct bio *inmate,
	147	struct dm_bio_prison_cell *cell_prealloc,
	148	struct dm_bio_prison_cell **cell_result)
	149	{
	150	int r;
	151	unsigned long flags;
4f81a417	152
a195db2d JT	153	spin_lock_irqsave(&prison->lock, flags);
	154	r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
	155	spin_unlock_irqrestore(&prison->lock, flags);
4f81a417	156
4f81a417 MS	157	return r;
4f81a417 MS	158	}
6beca5eb JT	159
	160	int dm_bio_detain(struct dm_bio_prison *prison,
	161	struct dm_cell_key *key,
	162	struct bio *inmate,
	163	struct dm_bio_prison_cell *cell_prealloc,
	164	struct dm_bio_prison_cell **cell_result)
	165	{
	166	return bio_detain(prison, key, inmate, cell_prealloc, cell_result);
	167	}
4f81a417 MS	168	EXPORT_SYMBOL_GPL(dm_bio_detain);
4f81a417 MS	169
c6b4fcba JT	170	int dm_get_cell(struct dm_bio_prison *prison,
	171	struct dm_cell_key *key,
	172	struct dm_bio_prison_cell *cell_prealloc,
	173	struct dm_bio_prison_cell **cell_result)
	174	{
	175	return bio_detain(prison, key, NULL, cell_prealloc, cell_result);
	176	}
	177	EXPORT_SYMBOL_GPL(dm_get_cell);
	178
4f81a417 MS	179	/*
	180	* @inmates must have been initialised prior to this call
	181	*/
a195db2d JT	182	static void __cell_release(struct dm_bio_prison *prison,
a195db2d JT	183	struct dm_bio_prison_cell *cell,
6beca5eb	184	struct bio_list *inmates)
4f81a417	185	{
a195db2d	186	rb_erase(&cell->node, &prison->cells);
4f81a417 MS	187
4f81a417 MS	188	if (inmates) {
6beca5eb JT	189	if (cell->holder)
6beca5eb JT	190	bio_list_add(inmates, cell->holder);
4f81a417 MS	191	bio_list_merge(inmates, &cell->bios);
4f81a417 MS	192	}
4f81a417 MS	193	}
4f81a417 MS	194
6beca5eb JT	195	void dm_cell_release(struct dm_bio_prison *prison,
	196	struct dm_bio_prison_cell *cell,
	197	struct bio_list *bios)
4f81a417 MS	198	{
4f81a417 MS	199	unsigned long flags;
4f81a417	200
a195db2d JT	201	spin_lock_irqsave(&prison->lock, flags);
	202	__cell_release(prison, cell, bios);
	203	spin_unlock_irqrestore(&prison->lock, flags);
4f81a417 MS	204	}
	205	EXPORT_SYMBOL_GPL(dm_cell_release);
	206
4f81a417 MS	207	/*
	208	* Sometimes we don't want the holder, just the additional bios.
	209	*/
a195db2d JT	210	static void __cell_release_no_holder(struct dm_bio_prison *prison,
a195db2d JT	211	struct dm_bio_prison_cell *cell,
6beca5eb	212	struct bio_list *inmates)
4f81a417	213	{
a195db2d	214	rb_erase(&cell->node, &prison->cells);
4f81a417	215	bio_list_merge(inmates, &cell->bios);
4f81a417 MS	216	}
4f81a417 MS	217
6beca5eb JT	218	void dm_cell_release_no_holder(struct dm_bio_prison *prison,
	219	struct dm_bio_prison_cell *cell,
	220	struct bio_list *inmates)
4f81a417 MS	221	{
4f81a417 MS	222	unsigned long flags;
4f81a417	223
a195db2d JT	224	spin_lock_irqsave(&prison->lock, flags);
	225	__cell_release_no_holder(prison, cell, inmates);
	226	spin_unlock_irqrestore(&prison->lock, flags);
4f81a417 MS	227	}
	228	EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);
	229
6beca5eb	230	void dm_cell_error(struct dm_bio_prison *prison,
af91805a	231	struct dm_bio_prison_cell *cell, int error)
4f81a417	232	{
4f81a417 MS	233	struct bio_list bios;
4f81a417 MS	234	struct bio *bio;
4f81a417 MS	235
4f81a417 MS	236	bio_list_init(&bios);
adcc4447	237	dm_cell_release(prison, cell, &bios);
4f81a417 MS	238
4f81a417 MS	239	while ((bio = bio_list_pop(&bios)))
af91805a	240	bio_endio(bio, error);
4f81a417 MS	241	}
	242	EXPORT_SYMBOL_GPL(dm_cell_error);
	243
2d759a46 JT	244	void dm_cell_visit_release(struct dm_bio_prison *prison,
	245	void (visit_fn)(void , struct dm_bio_prison_cell *),
	246	void *context,
	247	struct dm_bio_prison_cell *cell)
	248	{
	249	unsigned long flags;
	250
	251	spin_lock_irqsave(&prison->lock, flags);
	252	visit_fn(context, cell);
	253	rb_erase(&cell->node, &prison->cells);
	254	spin_unlock_irqrestore(&prison->lock, flags);
	255	}
	256	EXPORT_SYMBOL_GPL(dm_cell_visit_release);
	257
4f81a417 MS	258	/----------------------------------------------------------------/
	259
	260	#define DEFERRED_SET_SIZE 64
	261
	262	struct dm_deferred_entry {
	263	struct dm_deferred_set *ds;
	264	unsigned count;
	265	struct list_head work_items;
	266	};
	267
	268	struct dm_deferred_set {
	269	spinlock_t lock;
	270	unsigned current_entry;
	271	unsigned sweeper;
	272	struct dm_deferred_entry entries[DEFERRED_SET_SIZE];
	273	};
	274
	275	struct dm_deferred_set *dm_deferred_set_create(void)
	276	{
	277	int i;
	278	struct dm_deferred_set *ds;
	279
	280	ds = kmalloc(sizeof(*ds), GFP_KERNEL);
	281	if (!ds)
	282	return NULL;
	283
	284	spin_lock_init(&ds->lock);
	285	ds->current_entry = 0;
	286	ds->sweeper = 0;
	287	for (i = 0; i < DEFERRED_SET_SIZE; i++) {
	288	ds->entries[i].ds = ds;
	289	ds->entries[i].count = 0;
	290	INIT_LIST_HEAD(&ds->entries[i].work_items);
	291	}
	292
	293	return ds;
	294	}
	295	EXPORT_SYMBOL_GPL(dm_deferred_set_create);
	296
	297	void dm_deferred_set_destroy(struct dm_deferred_set *ds)
	298	{
	299	kfree(ds);
	300	}
	301	EXPORT_SYMBOL_GPL(dm_deferred_set_destroy);
	302
	303	struct dm_deferred_entry dm_deferred_entry_inc(struct dm_deferred_set ds)
	304	{
	305	unsigned long flags;
	306	struct dm_deferred_entry *entry;
	307
	308	spin_lock_irqsave(&ds->lock, flags);
	309	entry = ds->entries + ds->current_entry;
	310	entry->count++;
	311	spin_unlock_irqrestore(&ds->lock, flags);
	312
	313	return entry;
	314	}
	315	EXPORT_SYMBOL_GPL(dm_deferred_entry_inc);
	316
	317	static unsigned ds_next(unsigned index)
	318	{
	319	return (index + 1) % DEFERRED_SET_SIZE;
	320	}
	321
322	static void __sweep(struct dm_deferred_set ds, struct list_head head)
323	{
324	while ((ds->sweeper != ds->current_entry) &&
325	!ds->entries[ds->sweeper].count) {
326	list_splice_init(&ds->entries[ds->sweeper].work_items, head);
327	ds->sweeper = ds_next(ds->sweeper);
328	}
329
330	if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
331	list_splice_init(&ds->entries[ds->sweeper].work_items, head);
332	}
333
334	void dm_deferred_entry_dec(struct dm_deferred_entry entry, struct list_head head)
335	{
336	unsigned long flags;
337
338	spin_lock_irqsave(&entry->ds->lock, flags);
339	BUG_ON(!entry->count);
340	--entry->count;
341	__sweep(entry->ds, head);
342	spin_unlock_irqrestore(&entry->ds->lock, flags);
343	}
344	EXPORT_SYMBOL_GPL(dm_deferred_entry_dec);
345
346	/*
347	* Returns 1 if deferred or 0 if no pending items to delay job.
348	*/
349	int dm_deferred_set_add_work(struct dm_deferred_set ds, struct list_head work)
350	{
351	int r = 1;
352	unsigned long flags;
353	unsigned next_entry;
354
355	spin_lock_irqsave(&ds->lock, flags);
356	if ((ds->sweeper == ds->current_entry) &&
357	!ds->entries[ds->current_entry].count)
358	r = 0;
359	else {
360	list_add(work, &ds->entries[ds->current_entry].work_items);
361	next_entry = ds_next(ds->current_entry);
362	if (!ds->entries[next_entry].count)
363	ds->current_entry = next_entry;
364	}
365	spin_unlock_irqrestore(&ds->lock, flags);
366
367	return r;
368	}
369	EXPORT_SYMBOL_GPL(dm_deferred_set_add_work);
370
371	/----------------------------------------------------------------/
372
373	static int __init dm_bio_prison_init(void)
374	{
375	_cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
376	if (!_cell_cache)
377	return -ENOMEM;
378
379	return 0;
380	}
381
382	static void __exit dm_bio_prison_exit(void)
383	{
384	kmem_cache_destroy(_cell_cache);
385	_cell_cache = NULL;
386	}
387
388	/*
389	* module hooks
390	*/
391	module_init(dm_bio_prison_init);
392	module_exit(dm_bio_prison_exit);
393
394	MODULE_DESCRIPTION(DM_NAME " bio prison");
395	MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
396	MODULE_LICENSE("GPL");