[linux-block.git] / drivers / md / dm-bio-prison-v1.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2012 Red Hat, Inc.
 *
 * This file is released under the GPL.
 */

#include "dm.h"
#include "dm-bio-prison-v1.h"
#include "dm-bio-prison-v2.h"

#include <linux/spinlock.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/slab.h>

/*----------------------------------------------------------------*/

#define MIN_CELLS 1024

struct dm_bio_prison {
	spinlock_t lock;
	struct rb_root cells;
	mempool_t cell_pool;
};

static struct kmem_cache *_cell_cache;

/*----------------------------------------------------------------*/

/*
 * @nr_cells should be the number of cells you want in use _concurrently_.
 * Don't confuse it with the number of distinct keys.
 */
struct dm_bio_prison *dm_bio_prison_create(void)
{
	struct dm_bio_prison *prison = kzalloc(sizeof(*prison), GFP_KERNEL);
	int ret;

	if (!prison)
		return NULL;

	spin_lock_init(&prison->lock);

	ret = mempool_init_slab_pool(&prison->cell_pool, MIN_CELLS, _cell_cache);
	if (ret) {
		kfree(prison);
		return NULL;
	}

	prison->cells = RB_ROOT;

	return prison;
}
EXPORT_SYMBOL_GPL(dm_bio_prison_create);

void dm_bio_prison_destroy(struct dm_bio_prison *prison)
{
	mempool_exit(&prison->cell_pool);
	kfree(prison);
}
EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);

struct dm_bio_prison_cell *dm_bio_prison_alloc_cell(struct dm_bio_prison *prison, gfp_t gfp)
{
	return mempool_alloc(&prison->cell_pool, gfp);
}
EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);

void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
			     struct dm_bio_prison_cell *cell)
{
	mempool_free(cell, &prison->cell_pool);
}
EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);

static void __setup_new_cell(struct dm_cell_key *key,
			     struct bio *holder,
			     struct dm_bio_prison_cell *cell)
{
	memcpy(&cell->key, key, sizeof(cell->key));
	cell->holder = holder;
	bio_list_init(&cell->bios);
}

static int cmp_keys(struct dm_cell_key *lhs,
		    struct dm_cell_key *rhs)
{
	if (lhs->virtual < rhs->virtual)
		return -1;

	if (lhs->virtual > rhs->virtual)
		return 1;

	if (lhs->dev < rhs->dev)
		return -1;

	if (lhs->dev > rhs->dev)
		return 1;

	if (lhs->block_end <= rhs->block_begin)
		return -1;

	if (lhs->block_begin >= rhs->block_end)
		return 1;

	return 0;
}

static int __bio_detain(struct dm_bio_prison *prison,
			struct dm_cell_key *key,
			struct bio *inmate,
			struct dm_bio_prison_cell *cell_prealloc,
			struct dm_bio_prison_cell **cell_result)
{
	int r;
	struct rb_node **new = &prison->cells.rb_node, *parent = NULL;

	while (*new) {
		struct dm_bio_prison_cell *cell =
			rb_entry(*new, struct dm_bio_prison_cell, node);

		r = cmp_keys(key, &cell->key);

		parent = *new;
		if (r < 0)
			new = &((*new)->rb_left);
		else if (r > 0)
			new = &((*new)->rb_right);
		else {
			if (inmate)
				bio_list_add(&cell->bios, inmate);
			*cell_result = cell;
			return 1;
		}
	}

	__setup_new_cell(key, inmate, cell_prealloc);
	*cell_result = cell_prealloc;

	rb_link_node(&cell_prealloc->node, parent, new);
	rb_insert_color(&cell_prealloc->node, &prison->cells);

	return 0;
}

static int bio_detain(struct dm_bio_prison *prison,
		      struct dm_cell_key *key,
		      struct bio *inmate,
		      struct dm_bio_prison_cell *cell_prealloc,
		      struct dm_bio_prison_cell **cell_result)
{
	int r;

	spin_lock_irq(&prison->lock);
	r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
	spin_unlock_irq(&prison->lock);

	return r;
}

int dm_bio_detain(struct dm_bio_prison *prison,
		  struct dm_cell_key *key,
		  struct bio *inmate,
		  struct dm_bio_prison_cell *cell_prealloc,
		  struct dm_bio_prison_cell **cell_result)
{
	return bio_detain(prison, key, inmate, cell_prealloc, cell_result);
}
EXPORT_SYMBOL_GPL(dm_bio_detain);

int dm_get_cell(struct dm_bio_prison *prison,
		struct dm_cell_key *key,
		struct dm_bio_prison_cell *cell_prealloc,
		struct dm_bio_prison_cell **cell_result)
{
	return bio_detain(prison, key, NULL, cell_prealloc, cell_result);
}
EXPORT_SYMBOL_GPL(dm_get_cell);

/*
 * @inmates must have been initialised prior to this call
 */
static void __cell_release(struct dm_bio_prison *prison,
			   struct dm_bio_prison_cell *cell,
			   struct bio_list *inmates)
{
	rb_erase(&cell->node, &prison->cells);

	if (inmates) {
		if (cell->holder)
			bio_list_add(inmates, cell->holder);
		bio_list_merge(inmates, &cell->bios);
	}
}

void dm_cell_release(struct dm_bio_prison *prison,
		     struct dm_bio_prison_cell *cell,
		     struct bio_list *bios)
{
	spin_lock_irq(&prison->lock);
	__cell_release(prison, cell, bios);
	spin_unlock_irq(&prison->lock);
}
EXPORT_SYMBOL_GPL(dm_cell_release);

/*
 * Sometimes we don't want the holder, just the additional bios.
 */
static void __cell_release_no_holder(struct dm_bio_prison *prison,
				     struct dm_bio_prison_cell *cell,
				     struct bio_list *inmates)
{
	rb_erase(&cell->node, &prison->cells);
	bio_list_merge(inmates, &cell->bios);
}

void dm_cell_release_no_holder(struct dm_bio_prison *prison,
			       struct dm_bio_prison_cell *cell,
			       struct bio_list *inmates)
{
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	__cell_release_no_holder(prison, cell, inmates);
	spin_unlock_irqrestore(&prison->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);

void dm_cell_error(struct dm_bio_prison *prison,
		   struct dm_bio_prison_cell *cell, blk_status_t error)
{
	struct bio_list bios;
	struct bio *bio;

	bio_list_init(&bios);
	dm_cell_release(prison, cell, &bios);

	while ((bio = bio_list_pop(&bios))) {
		bio->bi_status = error;
		bio_endio(bio);
	}
}
EXPORT_SYMBOL_GPL(dm_cell_error);

void dm_cell_visit_release(struct dm_bio_prison *prison,
			   void (*visit_fn)(void *, struct dm_bio_prison_cell *),
			   void *context,
			   struct dm_bio_prison_cell *cell)
{
	spin_lock_irq(&prison->lock);
	visit_fn(context, cell);
	rb_erase(&cell->node, &prison->cells);
	spin_unlock_irq(&prison->lock);
}
EXPORT_SYMBOL_GPL(dm_cell_visit_release);

static int __promote_or_release(struct dm_bio_prison *prison,
				struct dm_bio_prison_cell *cell)
{
	if (bio_list_empty(&cell->bios)) {
		rb_erase(&cell->node, &prison->cells);
		return 1;
	}

	cell->holder = bio_list_pop(&cell->bios);
	return 0;
}

int dm_cell_promote_or_release(struct dm_bio_prison *prison,
			       struct dm_bio_prison_cell *cell)
{
	int r;

	spin_lock_irq(&prison->lock);
	r = __promote_or_release(prison, cell);
	spin_unlock_irq(&prison->lock);

	return r;
}
EXPORT_SYMBOL_GPL(dm_cell_promote_or_release);

/*----------------------------------------------------------------*/

#define DEFERRED_SET_SIZE 64

struct dm_deferred_entry {
	struct dm_deferred_set *ds;
	unsigned int count;
	struct list_head work_items;
};

struct dm_deferred_set {
	spinlock_t lock;
	unsigned int current_entry;
	unsigned int sweeper;
	struct dm_deferred_entry entries[DEFERRED_SET_SIZE];
};

struct dm_deferred_set *dm_deferred_set_create(void)
{
	int i;
	struct dm_deferred_set *ds;

	ds = kmalloc(sizeof(*ds), GFP_KERNEL);
	if (!ds)
		return NULL;

	spin_lock_init(&ds->lock);
	ds->current_entry = 0;
	ds->sweeper = 0;
	for (i = 0; i < DEFERRED_SET_SIZE; i++) {
		ds->entries[i].ds = ds;
		ds->entries[i].count = 0;
		INIT_LIST_HEAD(&ds->entries[i].work_items);
	}

	return ds;
}
EXPORT_SYMBOL_GPL(dm_deferred_set_create);

void dm_deferred_set_destroy(struct dm_deferred_set *ds)
{
	kfree(ds);
}
EXPORT_SYMBOL_GPL(dm_deferred_set_destroy);

struct dm_deferred_entry *dm_deferred_entry_inc(struct dm_deferred_set *ds)
{
	unsigned long flags;
	struct dm_deferred_entry *entry;

	spin_lock_irqsave(&ds->lock, flags);
	entry = ds->entries + ds->current_entry;
	entry->count++;
	spin_unlock_irqrestore(&ds->lock, flags);

	return entry;
}
EXPORT_SYMBOL_GPL(dm_deferred_entry_inc);

static unsigned int ds_next(unsigned int index)
{
	return (index + 1) % DEFERRED_SET_SIZE;
}

static void __sweep(struct dm_deferred_set *ds, struct list_head *head)
{
	while ((ds->sweeper != ds->current_entry) &&
	       !ds->entries[ds->sweeper].count) {
		list_splice_init(&ds->entries[ds->sweeper].work_items, head);
		ds->sweeper = ds_next(ds->sweeper);
	}

	if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
		list_splice_init(&ds->entries[ds->sweeper].work_items, head);
}

void dm_deferred_entry_dec(struct dm_deferred_entry *entry, struct list_head *head)
{
	unsigned long flags;

	spin_lock_irqsave(&entry->ds->lock, flags);
	BUG_ON(!entry->count);
	--entry->count;
	__sweep(entry->ds, head);
	spin_unlock_irqrestore(&entry->ds->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_deferred_entry_dec);

/*
 * Returns 1 if deferred or 0 if no pending items to delay job.
 */
int dm_deferred_set_add_work(struct dm_deferred_set *ds, struct list_head *work)
{
	int r = 1;
	unsigned int next_entry;

	spin_lock_irq(&ds->lock);
	if ((ds->sweeper == ds->current_entry) &&
	    !ds->entries[ds->current_entry].count)
		r = 0;
	else {
		list_add(work, &ds->entries[ds->current_entry].work_items);
		next_entry = ds_next(ds->current_entry);
		if (!ds->entries[next_entry].count)
			ds->current_entry = next_entry;
	}
	spin_unlock_irq(&ds->lock);

	return r;
}
EXPORT_SYMBOL_GPL(dm_deferred_set_add_work);

/*----------------------------------------------------------------*/

static int __init dm_bio_prison_init_v1(void)
{
	_cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
	if (!_cell_cache)
		return -ENOMEM;

	return 0;
}

static void dm_bio_prison_exit_v1(void)
{
	kmem_cache_destroy(_cell_cache);
	_cell_cache = NULL;
}

static int (*_inits[])(void) __initdata = {
	dm_bio_prison_init_v1,
	dm_bio_prison_init_v2,
};

static void (*_exits[])(void) = {
	dm_bio_prison_exit_v1,
	dm_bio_prison_exit_v2,
};

static int __init dm_bio_prison_init(void)
{
	const int count = ARRAY_SIZE(_inits);

	int r, i;

	for (i = 0; i < count; i++) {
		r = _inits[i]();
		if (r)
			goto bad;
	}

	return 0;

      bad:
	while (i--)
		_exits[i]();

	return r;
}

static void __exit dm_bio_prison_exit(void)
{
	int i = ARRAY_SIZE(_exits);

	while (i--)
		_exits[i]();
}

/*
 * module hooks
 */
module_init(dm_bio_prison_init);
module_exit(dm_bio_prison_exit);

MODULE_DESCRIPTION(DM_NAME " bio prison");
MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
MODULE_LICENSE("GPL");
Commit	Line	Data
3bd94003	1	// SPDX-License-Identifier: GPL-2.0-only
4f81a417 MS	2	/*
	3	* Copyright (C) 2012 Red Hat, Inc.
	4	*
	5	* This file is released under the GPL.
	6	*/
	7
	8	#include "dm.h"
742c8fdc JT	9	#include "dm-bio-prison-v1.h"
742c8fdc JT	10	#include "dm-bio-prison-v2.h"
4f81a417 MS	11
	12	#include <linux/spinlock.h>
	13	#include <linux/mempool.h>
	14	#include <linux/module.h>
	15	#include <linux/slab.h>
	16
	17	/----------------------------------------------------------------/
	18
a195db2d	19	#define MIN_CELLS 1024
adcc4447 HM	20
adcc4447 HM	21	struct dm_bio_prison {
a195db2d	22	spinlock_t lock;
a195db2d	23	struct rb_root cells;
72d711c8	24	mempool_t cell_pool;
4f81a417 MS	25	};
4f81a417 MS	26
4f81a417 MS	27	static struct kmem_cache *_cell_cache;
4f81a417 MS	28
a195db2d	29	/----------------------------------------------------------------/
adcc4447	30
4f81a417 MS	31	/*
	32	* @nr_cells should be the number of cells you want in use _concurrently_.
	33	* Don't confuse it with the number of distinct keys.
	34	*/
a195db2d	35	struct dm_bio_prison *dm_bio_prison_create(void)
4f81a417	36	{
d3775354	37	struct dm_bio_prison prison = kzalloc(sizeof(prison), GFP_KERNEL);
6f1c819c	38	int ret;
4f81a417 MS	39
	40	if (!prison)
	41	return NULL;
	42
a195db2d JT	43	spin_lock_init(&prison->lock);
a195db2d JT	44
6f1c819c KO	45	ret = mempool_init_slab_pool(&prison->cell_pool, MIN_CELLS, _cell_cache);
6f1c819c KO	46	if (ret) {
4f81a417 MS	47	kfree(prison);
	48	return NULL;
	49	}
	50
a195db2d	51	prison->cells = RB_ROOT;
4f81a417 MS	52
	53	return prison;
	54	}
	55	EXPORT_SYMBOL_GPL(dm_bio_prison_create);
	56
	57	void dm_bio_prison_destroy(struct dm_bio_prison *prison)
	58	{
6f1c819c	59	mempool_exit(&prison->cell_pool);
4f81a417 MS	60	kfree(prison);
	61	}
	62	EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);
	63
6beca5eb JT	64	struct dm_bio_prison_cell dm_bio_prison_alloc_cell(struct dm_bio_prison prison, gfp_t gfp)
6beca5eb JT	65	{
6f1c819c	66	return mempool_alloc(&prison->cell_pool, gfp);
6beca5eb JT	67	}
	68	EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);
	69
	70	void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
	71	struct dm_bio_prison_cell *cell)
	72	{
6f1c819c	73	mempool_free(cell, &prison->cell_pool);
6beca5eb JT	74	}
	75	EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);
	76
a195db2d JT	77	static void __setup_new_cell(struct dm_cell_key *key,
	78	struct bio *holder,
	79	struct dm_bio_prison_cell *cell)
4f81a417	80	{
8ca817c4 HM	81	memcpy(&cell->key, key, sizeof(cell->key));
	82	cell->holder = holder;
	83	bio_list_init(&cell->bios);
4f81a417 MS	84	}
4f81a417 MS	85
a195db2d JT	86	static int cmp_keys(struct dm_cell_key *lhs,
a195db2d JT	87	struct dm_cell_key *rhs)
4f81a417	88	{
a195db2d JT	89	if (lhs->virtual < rhs->virtual)
a195db2d JT	90	return -1;
4f81a417	91
a195db2d JT	92	if (lhs->virtual > rhs->virtual)
a195db2d JT	93	return 1;
adcc4447	94
a195db2d JT	95	if (lhs->dev < rhs->dev)
a195db2d JT	96	return -1;
4f81a417	97
a195db2d JT	98	if (lhs->dev > rhs->dev)
a195db2d JT	99	return 1;
4f81a417	100
5f274d88	101	if (lhs->block_end <= rhs->block_begin)
a195db2d	102	return -1;
4f81a417	103
5f274d88	104	if (lhs->block_begin >= rhs->block_end)
a195db2d JT	105	return 1;
	106
	107	return 0;
6beca5eb	108	}
4f81a417	109
a195db2d	110	static int __bio_detain(struct dm_bio_prison *prison,
6beca5eb JT	111	struct dm_cell_key *key,
	112	struct bio *inmate,
	113	struct dm_bio_prison_cell *cell_prealloc,
	114	struct dm_bio_prison_cell **cell_result)
	115	{
a195db2d JT	116	int r;
	117	struct rb_node *new = &prison->cells.rb_node, parent = NULL;
	118
	119	while (*new) {
	120	struct dm_bio_prison_cell *cell =
6e333d0b	121	rb_entry(*new, struct dm_bio_prison_cell, node);
a195db2d JT	122
	123	r = cmp_keys(key, &cell->key);
	124
	125	parent = *new;
	126	if (r < 0)
	127	new = &((*new)->rb_left);
	128	else if (r > 0)
	129	new = &((*new)->rb_right);
	130	else {
	131	if (inmate)
	132	bio_list_add(&cell->bios, inmate);
	133	*cell_result = cell;
	134	return 1;
	135	}
4f81a417 MS	136	}
4f81a417 MS	137
a195db2d	138	__setup_new_cell(key, inmate, cell_prealloc);
6beca5eb	139	*cell_result = cell_prealloc;
a195db2d JT	140
	141	rb_link_node(&cell_prealloc->node, parent, new);
	142	rb_insert_color(&cell_prealloc->node, &prison->cells);
	143
6beca5eb JT	144	return 0;
6beca5eb JT	145	}
4f81a417	146
6beca5eb JT	147	static int bio_detain(struct dm_bio_prison *prison,
	148	struct dm_cell_key *key,
	149	struct bio *inmate,
	150	struct dm_bio_prison_cell *cell_prealloc,
	151	struct dm_bio_prison_cell **cell_result)
	152	{
	153	int r;
4f81a417	154
235bc861	155	spin_lock_irq(&prison->lock);
a195db2d	156	r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
235bc861	157	spin_unlock_irq(&prison->lock);
4f81a417	158
4f81a417 MS	159	return r;
4f81a417 MS	160	}
6beca5eb JT	161
	162	int dm_bio_detain(struct dm_bio_prison *prison,
	163	struct dm_cell_key *key,
	164	struct bio *inmate,
	165	struct dm_bio_prison_cell *cell_prealloc,
	166	struct dm_bio_prison_cell **cell_result)
	167	{
	168	return bio_detain(prison, key, inmate, cell_prealloc, cell_result);
	169	}
4f81a417 MS	170	EXPORT_SYMBOL_GPL(dm_bio_detain);
4f81a417 MS	171
c6b4fcba JT	172	int dm_get_cell(struct dm_bio_prison *prison,
	173	struct dm_cell_key *key,
	174	struct dm_bio_prison_cell *cell_prealloc,
	175	struct dm_bio_prison_cell **cell_result)
	176	{
	177	return bio_detain(prison, key, NULL, cell_prealloc, cell_result);
	178	}
	179	EXPORT_SYMBOL_GPL(dm_get_cell);
	180
4f81a417 MS	181	/*
	182	* @inmates must have been initialised prior to this call
	183	*/
a195db2d JT	184	static void __cell_release(struct dm_bio_prison *prison,
a195db2d JT	185	struct dm_bio_prison_cell *cell,
6beca5eb	186	struct bio_list *inmates)
4f81a417	187	{
a195db2d	188	rb_erase(&cell->node, &prison->cells);
4f81a417 MS	189
4f81a417 MS	190	if (inmates) {
6beca5eb JT	191	if (cell->holder)
6beca5eb JT	192	bio_list_add(inmates, cell->holder);
4f81a417 MS	193	bio_list_merge(inmates, &cell->bios);
4f81a417 MS	194	}
4f81a417 MS	195	}
4f81a417 MS	196
6beca5eb JT	197	void dm_cell_release(struct dm_bio_prison *prison,
	198	struct dm_bio_prison_cell *cell,
	199	struct bio_list *bios)
4f81a417	200	{
235bc861	201	spin_lock_irq(&prison->lock);
a195db2d	202	__cell_release(prison, cell, bios);
235bc861	203	spin_unlock_irq(&prison->lock);
4f81a417 MS	204	}
	205	EXPORT_SYMBOL_GPL(dm_cell_release);
	206
4f81a417 MS	207	/*
	208	* Sometimes we don't want the holder, just the additional bios.
	209	*/
a195db2d JT	210	static void __cell_release_no_holder(struct dm_bio_prison *prison,
a195db2d JT	211	struct dm_bio_prison_cell *cell,
6beca5eb	212	struct bio_list *inmates)
4f81a417	213	{
a195db2d	214	rb_erase(&cell->node, &prison->cells);
4f81a417	215	bio_list_merge(inmates, &cell->bios);
4f81a417 MS	216	}
4f81a417 MS	217
6beca5eb JT	218	void dm_cell_release_no_holder(struct dm_bio_prison *prison,
	219	struct dm_bio_prison_cell *cell,
	220	struct bio_list *inmates)
4f81a417 MS	221	{
4f81a417 MS	222	unsigned long flags;
4f81a417	223
a195db2d JT	224	spin_lock_irqsave(&prison->lock, flags);
	225	__cell_release_no_holder(prison, cell, inmates);
	226	spin_unlock_irqrestore(&prison->lock, flags);
4f81a417 MS	227	}
	228	EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);
	229
6beca5eb	230	void dm_cell_error(struct dm_bio_prison *prison,
4e4cbee9	231	struct dm_bio_prison_cell *cell, blk_status_t error)
4f81a417	232	{
4f81a417 MS	233	struct bio_list bios;
4f81a417 MS	234	struct bio *bio;
4f81a417 MS	235
4f81a417 MS	236	bio_list_init(&bios);
adcc4447	237	dm_cell_release(prison, cell, &bios);
4f81a417	238
4246a0b6	239	while ((bio = bio_list_pop(&bios))) {
4e4cbee9	240	bio->bi_status = error;
4246a0b6 CH	241	bio_endio(bio);
4246a0b6 CH	242	}
4f81a417 MS	243	}
	244	EXPORT_SYMBOL_GPL(dm_cell_error);
	245
2d759a46 JT	246	void dm_cell_visit_release(struct dm_bio_prison *prison,
	247	void (visit_fn)(void , struct dm_bio_prison_cell *),
	248	void *context,
	249	struct dm_bio_prison_cell *cell)
	250	{
235bc861	251	spin_lock_irq(&prison->lock);
2d759a46 JT	252	visit_fn(context, cell);
2d759a46 JT	253	rb_erase(&cell->node, &prison->cells);
235bc861	254	spin_unlock_irq(&prison->lock);
2d759a46 JT	255	}
	256	EXPORT_SYMBOL_GPL(dm_cell_visit_release);
	257
3cdf93f9 JT	258	static int __promote_or_release(struct dm_bio_prison *prison,
	259	struct dm_bio_prison_cell *cell)
	260	{
	261	if (bio_list_empty(&cell->bios)) {
	262	rb_erase(&cell->node, &prison->cells);
	263	return 1;
	264	}
	265
	266	cell->holder = bio_list_pop(&cell->bios);
	267	return 0;
	268	}
	269
	270	int dm_cell_promote_or_release(struct dm_bio_prison *prison,
	271	struct dm_bio_prison_cell *cell)
	272	{
	273	int r;
3cdf93f9	274
235bc861	275	spin_lock_irq(&prison->lock);
3cdf93f9	276	r = __promote_or_release(prison, cell);
235bc861	277	spin_unlock_irq(&prison->lock);
3cdf93f9 JT	278
	279	return r;
	280	}
	281	EXPORT_SYMBOL_GPL(dm_cell_promote_or_release);
	282
4f81a417 MS	283	/----------------------------------------------------------------/
	284
	285	#define DEFERRED_SET_SIZE 64
	286
	287	struct dm_deferred_entry {
	288	struct dm_deferred_set *ds;
86a3238c	289	unsigned int count;
4f81a417 MS	290	struct list_head work_items;
	291	};
	292
	293	struct dm_deferred_set {
	294	spinlock_t lock;
86a3238c HM	295	unsigned int current_entry;
86a3238c HM	296	unsigned int sweeper;
4f81a417 MS	297	struct dm_deferred_entry entries[DEFERRED_SET_SIZE];
	298	};
	299
	300	struct dm_deferred_set *dm_deferred_set_create(void)
	301	{
	302	int i;
	303	struct dm_deferred_set *ds;
	304
	305	ds = kmalloc(sizeof(*ds), GFP_KERNEL);
	306	if (!ds)
	307	return NULL;
	308
	309	spin_lock_init(&ds->lock);
	310	ds->current_entry = 0;
	311	ds->sweeper = 0;
	312	for (i = 0; i < DEFERRED_SET_SIZE; i++) {
	313	ds->entries[i].ds = ds;
	314	ds->entries[i].count = 0;
	315	INIT_LIST_HEAD(&ds->entries[i].work_items);
	316	}
	317
	318	return ds;
	319	}
	320	EXPORT_SYMBOL_GPL(dm_deferred_set_create);
	321
	322	void dm_deferred_set_destroy(struct dm_deferred_set *ds)
	323	{
	324	kfree(ds);
	325	}
	326	EXPORT_SYMBOL_GPL(dm_deferred_set_destroy);
	327
	328	struct dm_deferred_entry dm_deferred_entry_inc(struct dm_deferred_set ds)
	329	{
	330	unsigned long flags;
	331	struct dm_deferred_entry *entry;
	332
	333	spin_lock_irqsave(&ds->lock, flags);
	334	entry = ds->entries + ds->current_entry;
	335	entry->count++;
	336	spin_unlock_irqrestore(&ds->lock, flags);
	337
	338	return entry;
	339	}
	340	EXPORT_SYMBOL_GPL(dm_deferred_entry_inc);
	341
86a3238c	342	static unsigned int ds_next(unsigned int index)
4f81a417 MS	343	{
	344	return (index + 1) % DEFERRED_SET_SIZE;
	345	}
	346
	347	static void __sweep(struct dm_deferred_set ds, struct list_head head)
	348	{
	349	while ((ds->sweeper != ds->current_entry) &&
	350	!ds->entries[ds->sweeper].count) {
	351	list_splice_init(&ds->entries[ds->sweeper].work_items, head);
	352	ds->sweeper = ds_next(ds->sweeper);
	353	}
	354
	355	if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
	356	list_splice_init(&ds->entries[ds->sweeper].work_items, head);
	357	}
	358
	359	void dm_deferred_entry_dec(struct dm_deferred_entry entry, struct list_head head)
	360	{
	361	unsigned long flags;
	362
	363	spin_lock_irqsave(&entry->ds->lock, flags);
	364	BUG_ON(!entry->count);
	365	--entry->count;
	366	__sweep(entry->ds, head);
	367	spin_unlock_irqrestore(&entry->ds->lock, flags);
	368	}
	369	EXPORT_SYMBOL_GPL(dm_deferred_entry_dec);
	370
	371	/*
	372	* Returns 1 if deferred or 0 if no pending items to delay job.
	373	*/
	374	int dm_deferred_set_add_work(struct dm_deferred_set ds, struct list_head work)
	375	{
	376	int r = 1;
86a3238c	377	unsigned int next_entry;
4f81a417	378
235bc861	379	spin_lock_irq(&ds->lock);
4f81a417 MS	380	if ((ds->sweeper == ds->current_entry) &&
	381	!ds->entries[ds->current_entry].count)
	382	r = 0;
	383	else {
	384	list_add(work, &ds->entries[ds->current_entry].work_items);
	385	next_entry = ds_next(ds->current_entry);
	386	if (!ds->entries[next_entry].count)
	387	ds->current_entry = next_entry;
	388	}
235bc861	389	spin_unlock_irq(&ds->lock);
4f81a417 MS	390
	391	return r;
	392	}
	393	EXPORT_SYMBOL_GPL(dm_deferred_set_add_work);
	394
	395	/----------------------------------------------------------------/
	396
742c8fdc	397	static int __init dm_bio_prison_init_v1(void)
4f81a417 MS	398	{
	399	_cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
	400	if (!_cell_cache)
	401	return -ENOMEM;
	402
	403	return 0;
	404	}
	405
742c8fdc	406	static void dm_bio_prison_exit_v1(void)
4f81a417 MS	407	{
	408	kmem_cache_destroy(_cell_cache);
	409	_cell_cache = NULL;
	410	}
	411
742c8fdc JT	412	static int (*_inits[])(void) __initdata = {
	413	dm_bio_prison_init_v1,
	414	dm_bio_prison_init_v2,
	415	};
	416
	417	static void (*_exits[])(void) = {
	418	dm_bio_prison_exit_v1,
	419	dm_bio_prison_exit_v2,
	420	};
	421
	422	static int __init dm_bio_prison_init(void)
	423	{
	424	const int count = ARRAY_SIZE(_inits);
	425
	426	int r, i;
	427
	428	for (i = 0; i < count; i++) {
	429	r = _inits[i]();
	430	if (r)
	431	goto bad;
	432	}
	433
	434	return 0;
	435
	436	bad:
	437	while (i--)
	438	_exits[i]();
	439
	440	return r;
	441	}
	442
	443	static void __exit dm_bio_prison_exit(void)
	444	{
	445	int i = ARRAY_SIZE(_exits);
	446
	447	while (i--)
	448	_exits[i]();
	449	}
	450
4f81a417 MS	451	/*
	452	* module hooks
	453	*/
	454	module_init(dm_bio_prison_init);
	455	module_exit(dm_bio_prison_exit);
	456
	457	MODULE_DESCRIPTION(DM_NAME " bio prison");
	458	MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
	459	MODULE_LICENSE("GPL");