[linux-2.6-block.git] / block / blk-mq-tag.c

/*
 * Tag allocation using scalable bitmaps. Uses active queue tracking to support
 * fairer distribution of tags between multiple submitters when a shared tag map
 * is used.
 *
 * Copyright (C) 2013-2014 Jens Axboe
 */
#include <linux/kernel.h>
#include <linux/module.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"

bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
{
	if (!tags)
		return true;

	return sbitmap_any_bit_clear(&tags->bitmap_tags.sb);
}

/*
 * If a previously inactive queue goes active, bump the active user count.
 */
bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
{
	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) &&
	    !test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
		atomic_inc(&hctx->tags->active_queues);

	return true;
}

/*
 * Wakeup all potentially sleeping on tags
 */
void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve)
{
	sbitmap_queue_wake_all(&tags->bitmap_tags);
	if (include_reserve)
		sbitmap_queue_wake_all(&tags->breserved_tags);
}

/*
 * If a previously busy queue goes inactive, potential waiters could now
 * be allowed to queue. Wake them up and check.
 */
void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_tags *tags = hctx->tags;

	if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
		return;

	atomic_dec(&tags->active_queues);

	blk_mq_tag_wakeup_all(tags, false);
}

/*
 * For shared tag users, we track the number of currently active users
 * and attempt to provide a fair share of the tag depth for each of them.
 */
static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
				  struct sbitmap_queue *bt)
{
	unsigned int depth, users;

	if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_SHARED))
		return true;
	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
		return true;

	/*
	 * Don't try dividing an ant
	 */
	if (bt->sb.depth == 1)
		return true;

	users = atomic_read(&hctx->tags->active_queues);
	if (!users)
		return true;

	/*
	 * Allow at least some tags
	 */
	depth = max((bt->sb.depth + users - 1) / users, 4U);
	return atomic_read(&hctx->nr_active) < depth;
}

static int __blk_mq_get_tag(struct blk_mq_alloc_data *data,
			    struct sbitmap_queue *bt)
{
	if (!(data->flags & BLK_MQ_REQ_INTERNAL) &&
	    !hctx_may_queue(data->hctx, bt))
		return -1;
	if (data->shallow_depth)
		return __sbitmap_queue_get_shallow(bt, data->shallow_depth);
	else
		return __sbitmap_queue_get(bt);
}

unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
{
	struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
	struct sbitmap_queue *bt;
	struct sbq_wait_state *ws;
	DEFINE_WAIT(wait);
	unsigned int tag_offset;
	bool drop_ctx;
	int tag;

	if (data->flags & BLK_MQ_REQ_RESERVED) {
		if (unlikely(!tags->nr_reserved_tags)) {
			WARN_ON_ONCE(1);
			return BLK_MQ_TAG_FAIL;
		}
		bt = &tags->breserved_tags;
		tag_offset = 0;
	} else {
		bt = &tags->bitmap_tags;
		tag_offset = tags->nr_reserved_tags;
	}

	tag = __blk_mq_get_tag(data, bt);
	if (tag != -1)
		goto found_tag;

	if (data->flags & BLK_MQ_REQ_NOWAIT)
		return BLK_MQ_TAG_FAIL;

	ws = bt_wait_ptr(bt, data->hctx);
	drop_ctx = data->ctx == NULL;
	do {
		struct sbitmap_queue *bt_prev;

		/*
		 * We're out of tags on this hardware queue, kick any
		 * pending IO submits before going to sleep waiting for
		 * some to complete.
		 */
		blk_mq_run_hw_queue(data->hctx, false);

		/*
		 * Retry tag allocation after running the hardware queue,
		 * as running the queue may also have found completions.
		 */
		tag = __blk_mq_get_tag(data, bt);
		if (tag != -1)
			break;

		prepare_to_wait_exclusive(&ws->wait, &wait,
						TASK_UNINTERRUPTIBLE);

		tag = __blk_mq_get_tag(data, bt);
		if (tag != -1)
			break;

		if (data->ctx)
			blk_mq_put_ctx(data->ctx);

		bt_prev = bt;
		io_schedule();

		data->ctx = blk_mq_get_ctx(data->q);
		data->hctx = blk_mq_map_queue(data->q, data->ctx->cpu);
		tags = blk_mq_tags_from_data(data);
		if (data->flags & BLK_MQ_REQ_RESERVED)
			bt = &tags->breserved_tags;
		else
			bt = &tags->bitmap_tags;

		finish_wait(&ws->wait, &wait);

		/*
		 * If destination hw queue is changed, fake wake up on
		 * previous queue for compensating the wake up miss, so
		 * other allocations on previous queue won't be starved.
		 */
		if (bt != bt_prev)
			sbitmap_queue_wake_up(bt_prev);

		ws = bt_wait_ptr(bt, data->hctx);
	} while (1);

	if (drop_ctx && data->ctx)
		blk_mq_put_ctx(data->ctx);

	finish_wait(&ws->wait, &wait);

found_tag:
	return tag + tag_offset;
}

void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, struct blk_mq_tags *tags,
		    struct blk_mq_ctx *ctx, unsigned int tag)
{
	if (!blk_mq_tag_is_reserved(tags, tag)) {
		const int real_tag = tag - tags->nr_reserved_tags;

		BUG_ON(real_tag >= tags->nr_tags);
		sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu);
	} else {
		BUG_ON(tag >= tags->nr_reserved_tags);
		sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu);
	}
}

struct bt_iter_data {
	struct blk_mq_hw_ctx *hctx;
	busy_iter_fn *fn;
	void *data;
	bool reserved;
};

static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
{
	struct bt_iter_data *iter_data = data;
	struct blk_mq_hw_ctx *hctx = iter_data->hctx;
	struct blk_mq_tags *tags = hctx->tags;
	bool reserved = iter_data->reserved;
	struct request *rq;

	if (!reserved)
		bitnr += tags->nr_reserved_tags;
	rq = tags->rqs[bitnr];

	/*
	 * We can hit rq == NULL here, because the tagging functions
	 * test and set the bit before assining ->rqs[].
	 */
	if (rq && rq->q == hctx->queue)
		iter_data->fn(hctx, rq, iter_data->data, reserved);
	return true;
}

static void bt_for_each(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt,
			busy_iter_fn *fn, void *data, bool reserved)
{
	struct bt_iter_data iter_data = {
		.hctx = hctx,
		.fn = fn,
		.data = data,
		.reserved = reserved,
	};

	sbitmap_for_each_set(&bt->sb, bt_iter, &iter_data);
}

struct bt_tags_iter_data {
	struct blk_mq_tags *tags;
	busy_tag_iter_fn *fn;
	void *data;
	bool reserved;
};

static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
{
	struct bt_tags_iter_data *iter_data = data;
	struct blk_mq_tags *tags = iter_data->tags;
	bool reserved = iter_data->reserved;
	struct request *rq;

	if (!reserved)
		bitnr += tags->nr_reserved_tags;

	/*
	 * We can hit rq == NULL here, because the tagging functions
	 * test and set the bit before assining ->rqs[].
	 */
	rq = tags->rqs[bitnr];
	if (rq && blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
		iter_data->fn(rq, iter_data->data, reserved);

	return true;
}

static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt,
			     busy_tag_iter_fn *fn, void *data, bool reserved)
{
	struct bt_tags_iter_data iter_data = {
		.tags = tags,
		.fn = fn,
		.data = data,
		.reserved = reserved,
	};

	if (tags->rqs)
		sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data);
}

static void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags,
		busy_tag_iter_fn *fn, void *priv)
{
	if (tags->nr_reserved_tags)
		bt_tags_for_each(tags, &tags->breserved_tags, fn, priv, true);
	bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, false);
}

void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
		busy_tag_iter_fn *fn, void *priv)
{
	int i;

	for (i = 0; i < tagset->nr_hw_queues; i++) {
		if (tagset->tags && tagset->tags[i])
			blk_mq_all_tag_busy_iter(tagset->tags[i], fn, priv);
	}
}
EXPORT_SYMBOL(blk_mq_tagset_busy_iter);

void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
		void *priv)
{
	struct blk_mq_hw_ctx *hctx;
	int i;


	queue_for_each_hw_ctx(q, hctx, i) {
		struct blk_mq_tags *tags = hctx->tags;

		/*
		 * If not software queues are currently mapped to this
		 * hardware queue, there's nothing to check
		 */
		if (!blk_mq_hw_queue_mapped(hctx))
			continue;

		if (tags->nr_reserved_tags)
			bt_for_each(hctx, &tags->breserved_tags, fn, priv, true);
		bt_for_each(hctx, &tags->bitmap_tags, fn, priv, false);
	}

}

static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth,
		    bool round_robin, int node)
{
	return sbitmap_queue_init_node(bt, depth, -1, round_robin, GFP_KERNEL,
				       node);
}

static struct blk_mq_tags *blk_mq_init_bitmap_tags(struct blk_mq_tags *tags,
						   int node, int alloc_policy)
{
	unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;
	bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;

	if (bt_alloc(&tags->bitmap_tags, depth, round_robin, node))
		goto free_tags;
	if (bt_alloc(&tags->breserved_tags, tags->nr_reserved_tags, round_robin,
		     node))
		goto free_bitmap_tags;

	return tags;
free_bitmap_tags:
	sbitmap_queue_free(&tags->bitmap_tags);
free_tags:
	kfree(tags);
	return NULL;
}

struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
				     unsigned int reserved_tags,
				     int node, int alloc_policy)
{
	struct blk_mq_tags *tags;

	if (total_tags > BLK_MQ_TAG_MAX) {
		pr_err("blk-mq: tag depth too large\n");
		return NULL;
	}

	tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node);
	if (!tags)
		return NULL;

	tags->nr_tags = total_tags;
	tags->nr_reserved_tags = reserved_tags;

	return blk_mq_init_bitmap_tags(tags, node, alloc_policy);
}

void blk_mq_free_tags(struct blk_mq_tags *tags)
{
	sbitmap_queue_free(&tags->bitmap_tags);
	sbitmap_queue_free(&tags->breserved_tags);
	kfree(tags);
}

int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
			    struct blk_mq_tags **tagsptr, unsigned int tdepth,
			    bool can_grow)
{
	struct blk_mq_tags *tags = *tagsptr;

	if (tdepth <= tags->nr_reserved_tags)
		return -EINVAL;

	tdepth -= tags->nr_reserved_tags;

	/*
	 * If we are allowed to grow beyond the original size, allocate
	 * a new set of tags before freeing the old one.
	 */
	if (tdepth > tags->nr_tags) {
		struct blk_mq_tag_set *set = hctx->queue->tag_set;
		struct blk_mq_tags *new;
		bool ret;

		if (!can_grow)
			return -EINVAL;

		/*
		 * We need some sort of upper limit, set it high enough that
		 * no valid use cases should require more.
		 */
		if (tdepth > 16 * BLKDEV_MAX_RQ)
			return -EINVAL;

		new = blk_mq_alloc_rq_map(set, hctx->queue_num, tdepth, 0);
		if (!new)
			return -ENOMEM;
		ret = blk_mq_alloc_rqs(set, new, hctx->queue_num, tdepth);
		if (ret) {
			blk_mq_free_rq_map(new);
			return -ENOMEM;
		}

		blk_mq_free_rqs(set, *tagsptr, hctx->queue_num);
		blk_mq_free_rq_map(*tagsptr);
		*tagsptr = new;
	} else {
		/*
		 * Don't need (or can't) update reserved tags here, they
		 * remain static and should never need resizing.
		 */
		sbitmap_queue_resize(&tags->bitmap_tags, tdepth);
	}

	return 0;
}

/**
 * blk_mq_unique_tag() - return a tag that is unique queue-wide
 * @rq: request for which to compute a unique tag
 *
 * The tag field in struct request is unique per hardware queue but not over
 * all hardware queues. Hence this function that returns a tag with the
 * hardware context index in the upper bits and the per hardware queue tag in
 * the lower bits.
 *
 * Note: When called for a request that is queued on a non-multiqueue request
 * queue, the hardware context index is set to zero.
 */
u32 blk_mq_unique_tag(struct request *rq)
{
	struct request_queue *q = rq->q;
	struct blk_mq_hw_ctx *hctx;
	int hwq = 0;

	if (q->mq_ops) {
		hctx = blk_mq_map_queue(q, rq->mq_ctx->cpu);
		hwq = hctx->queue_num;
	}

	return (hwq << BLK_MQ_UNIQUE_TAG_BITS) |
		(rq->tag & BLK_MQ_UNIQUE_TAG_MASK);
}
EXPORT_SYMBOL(blk_mq_unique_tag);
Commit	Line	Data
	1	/*
	2	* Tag allocation using scalable bitmaps. Uses active queue tracking to support
	3	* fairer distribution of tags between multiple submitters when a shared tag map
	4	* is used.
	5	*
	6	* Copyright (C) 2013-2014 Jens Axboe
	7	*/
	8	#include <linux/kernel.h>
	9	#include <linux/module.h>
	10
	11	#include <linux/blk-mq.h>
	12	#include "blk.h"
	13	#include "blk-mq.h"
	14	#include "blk-mq-tag.h"
	15
	16	bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
	17	{
	18	if (!tags)
	19	return true;
	20
	21	return sbitmap_any_bit_clear(&tags->bitmap_tags.sb);
	22	}
	23
	24	/*
	25	* If a previously inactive queue goes active, bump the active user count.
	26	*/
	27	bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
	28	{
	29	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) &&
	30	!test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
	31	atomic_inc(&hctx->tags->active_queues);
	32
	33	return true;
	34	}
	35
	36	/*
	37	* Wakeup all potentially sleeping on tags
	38	*/
	39	void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve)
	40	{
	41	sbitmap_queue_wake_all(&tags->bitmap_tags);
	42	if (include_reserve)
	43	sbitmap_queue_wake_all(&tags->breserved_tags);
	44	}
	45
	46	/*
	47	* If a previously busy queue goes inactive, potential waiters could now
	48	* be allowed to queue. Wake them up and check.
	49	*/
	50	void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
	51	{
	52	struct blk_mq_tags *tags = hctx->tags;
	53
	54	if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
	55	return;
	56
	57	atomic_dec(&tags->active_queues);
	58
	59	blk_mq_tag_wakeup_all(tags, false);
	60	}
	61
	62	/*
	63	* For shared tag users, we track the number of currently active users
	64	* and attempt to provide a fair share of the tag depth for each of them.
	65	*/
	66	static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
	67	struct sbitmap_queue *bt)
	68	{
	69	unsigned int depth, users;
	70
	71	if (!hctx \|\| !(hctx->flags & BLK_MQ_F_TAG_SHARED))
	72	return true;
	73	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
	74	return true;
	75
	76	/*
	77	* Don't try dividing an ant
	78	*/
	79	if (bt->sb.depth == 1)
	80	return true;
	81
	82	users = atomic_read(&hctx->tags->active_queues);
	83	if (!users)
	84	return true;
	85
	86	/*
	87	* Allow at least some tags
	88	*/
	89	depth = max((bt->sb.depth + users - 1) / users, 4U);
	90	return atomic_read(&hctx->nr_active) < depth;
	91	}
	92
	93	static int __blk_mq_get_tag(struct blk_mq_alloc_data *data,
	94	struct sbitmap_queue *bt)
	95	{
	96	if (!(data->flags & BLK_MQ_REQ_INTERNAL) &&
	97	!hctx_may_queue(data->hctx, bt))
	98	return -1;
	99	if (data->shallow_depth)
	100	return __sbitmap_queue_get_shallow(bt, data->shallow_depth);
	101	else
	102	return __sbitmap_queue_get(bt);
	103	}
	104
	105	unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
	106	{
	107	struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
	108	struct sbitmap_queue *bt;
	109	struct sbq_wait_state *ws;
	110	DEFINE_WAIT(wait);
	111	unsigned int tag_offset;
	112	bool drop_ctx;
	113	int tag;
	114
	115	if (data->flags & BLK_MQ_REQ_RESERVED) {
	116	if (unlikely(!tags->nr_reserved_tags)) {
	117	WARN_ON_ONCE(1);
	118	return BLK_MQ_TAG_FAIL;
	119	}
	120	bt = &tags->breserved_tags;
	121	tag_offset = 0;
	122	} else {
	123	bt = &tags->bitmap_tags;
	124	tag_offset = tags->nr_reserved_tags;
	125	}
	126
	127	tag = __blk_mq_get_tag(data, bt);
	128	if (tag != -1)
	129	goto found_tag;
	130
	131	if (data->flags & BLK_MQ_REQ_NOWAIT)
	132	return BLK_MQ_TAG_FAIL;
	133
	134	ws = bt_wait_ptr(bt, data->hctx);
	135	drop_ctx = data->ctx == NULL;
	136	do {
	137	struct sbitmap_queue *bt_prev;
	138
	139	/*
	140	* We're out of tags on this hardware queue, kick any
	141	* pending IO submits before going to sleep waiting for
	142	* some to complete.
	143	*/
	144	blk_mq_run_hw_queue(data->hctx, false);
	145
	146	/*
	147	* Retry tag allocation after running the hardware queue,
	148	* as running the queue may also have found completions.
	149	*/
	150	tag = __blk_mq_get_tag(data, bt);
	151	if (tag != -1)
	152	break;
	153
	154	prepare_to_wait_exclusive(&ws->wait, &wait,
	155	TASK_UNINTERRUPTIBLE);
	156
	157	tag = __blk_mq_get_tag(data, bt);
	158	if (tag != -1)
	159	break;
	160
	161	if (data->ctx)
	162	blk_mq_put_ctx(data->ctx);
	163
	164	bt_prev = bt;
	165	io_schedule();
	166
	167	data->ctx = blk_mq_get_ctx(data->q);
	168	data->hctx = blk_mq_map_queue(data->q, data->ctx->cpu);
	169	tags = blk_mq_tags_from_data(data);
	170	if (data->flags & BLK_MQ_REQ_RESERVED)
	171	bt = &tags->breserved_tags;
	172	else
	173	bt = &tags->bitmap_tags;
	174
	175	finish_wait(&ws->wait, &wait);
	176
	177	/*
	178	* If destination hw queue is changed, fake wake up on
	179	* previous queue for compensating the wake up miss, so
	180	* other allocations on previous queue won't be starved.
	181	*/
	182	if (bt != bt_prev)
	183	sbitmap_queue_wake_up(bt_prev);
	184
	185	ws = bt_wait_ptr(bt, data->hctx);
	186	} while (1);
	187
	188	if (drop_ctx && data->ctx)
	189	blk_mq_put_ctx(data->ctx);
	190
	191	finish_wait(&ws->wait, &wait);
	192
	193	found_tag:
	194	return tag + tag_offset;
	195	}
	196
	197	void blk_mq_put_tag(struct blk_mq_hw_ctx hctx, struct blk_mq_tags tags,
	198	struct blk_mq_ctx *ctx, unsigned int tag)
	199	{
	200	if (!blk_mq_tag_is_reserved(tags, tag)) {
	201	const int real_tag = tag - tags->nr_reserved_tags;
	202
	203	BUG_ON(real_tag >= tags->nr_tags);
	204	sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu);
	205	} else {
	206	BUG_ON(tag >= tags->nr_reserved_tags);
	207	sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu);
	208	}
	209	}
	210
	211	struct bt_iter_data {
	212	struct blk_mq_hw_ctx *hctx;
	213	busy_iter_fn *fn;
	214	void *data;
	215	bool reserved;
	216	};
	217
	218	static bool bt_iter(struct sbitmap bitmap, unsigned int bitnr, void data)
	219	{
	220	struct bt_iter_data *iter_data = data;
	221	struct blk_mq_hw_ctx *hctx = iter_data->hctx;
	222	struct blk_mq_tags *tags = hctx->tags;
	223	bool reserved = iter_data->reserved;
	224	struct request *rq;
	225
	226	if (!reserved)
	227	bitnr += tags->nr_reserved_tags;
	228	rq = tags->rqs[bitnr];
	229
	230	/*
	231	* We can hit rq == NULL here, because the tagging functions
	232	* test and set the bit before assining ->rqs[].
	233	*/
	234	if (rq && rq->q == hctx->queue)
	235	iter_data->fn(hctx, rq, iter_data->data, reserved);
	236	return true;
	237	}
	238
	239	static void bt_for_each(struct blk_mq_hw_ctx hctx, struct sbitmap_queue bt,
	240	busy_iter_fn fn, void data, bool reserved)
	241	{
	242	struct bt_iter_data iter_data = {
	243	.hctx = hctx,
	244	.fn = fn,
	245	.data = data,
	246	.reserved = reserved,
	247	};
	248
	249	sbitmap_for_each_set(&bt->sb, bt_iter, &iter_data);
	250	}
	251
	252	struct bt_tags_iter_data {
	253	struct blk_mq_tags *tags;
	254	busy_tag_iter_fn *fn;
	255	void *data;
	256	bool reserved;
	257	};
	258
	259	static bool bt_tags_iter(struct sbitmap bitmap, unsigned int bitnr, void data)
	260	{
	261	struct bt_tags_iter_data *iter_data = data;
	262	struct blk_mq_tags *tags = iter_data->tags;
	263	bool reserved = iter_data->reserved;
	264	struct request *rq;
	265
	266	if (!reserved)
	267	bitnr += tags->nr_reserved_tags;
	268
	269	/*
	270	* We can hit rq == NULL here, because the tagging functions
	271	* test and set the bit before assining ->rqs[].
	272	*/
	273	rq = tags->rqs[bitnr];
	274	if (rq && blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
	275	iter_data->fn(rq, iter_data->data, reserved);
	276
	277	return true;
	278	}
	279
	280	static void bt_tags_for_each(struct blk_mq_tags tags, struct sbitmap_queue bt,
	281	busy_tag_iter_fn fn, void data, bool reserved)
	282	{
	283	struct bt_tags_iter_data iter_data = {
	284	.tags = tags,
	285	.fn = fn,
	286	.data = data,
	287	.reserved = reserved,
	288	};
	289
	290	if (tags->rqs)
	291	sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data);
	292	}
	293
	294	static void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags,
	295	busy_tag_iter_fn fn, void priv)
	296	{
	297	if (tags->nr_reserved_tags)
	298	bt_tags_for_each(tags, &tags->breserved_tags, fn, priv, true);
	299	bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, false);
	300	}
	301
	302	void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
	303	busy_tag_iter_fn fn, void priv)
	304	{
	305	int i;
	306
	307	for (i = 0; i < tagset->nr_hw_queues; i++) {
	308	if (tagset->tags && tagset->tags[i])
	309	blk_mq_all_tag_busy_iter(tagset->tags[i], fn, priv);
	310	}
	311	}
	312	EXPORT_SYMBOL(blk_mq_tagset_busy_iter);
	313
	314	void blk_mq_queue_tag_busy_iter(struct request_queue q, busy_iter_fn fn,
	315	void *priv)
	316	{
	317	struct blk_mq_hw_ctx *hctx;
	318	int i;
	319
	320
	321	queue_for_each_hw_ctx(q, hctx, i) {
	322	struct blk_mq_tags *tags = hctx->tags;
	323
	324	/*
	325	* If not software queues are currently mapped to this
	326	* hardware queue, there's nothing to check
	327	*/
	328	if (!blk_mq_hw_queue_mapped(hctx))
	329	continue;
	330
	331	if (tags->nr_reserved_tags)
	332	bt_for_each(hctx, &tags->breserved_tags, fn, priv, true);
	333	bt_for_each(hctx, &tags->bitmap_tags, fn, priv, false);
	334	}
	335
	336	}
	337
	338	static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth,
	339	bool round_robin, int node)
	340	{
	341	return sbitmap_queue_init_node(bt, depth, -1, round_robin, GFP_KERNEL,
	342	node);
	343	}
	344
	345	static struct blk_mq_tags blk_mq_init_bitmap_tags(struct blk_mq_tags tags,
	346	int node, int alloc_policy)
	347	{
	348	unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;
	349	bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;
	350
	351	if (bt_alloc(&tags->bitmap_tags, depth, round_robin, node))
	352	goto free_tags;
	353	if (bt_alloc(&tags->breserved_tags, tags->nr_reserved_tags, round_robin,
	354	node))
	355	goto free_bitmap_tags;
	356
	357	return tags;
	358	free_bitmap_tags:
	359	sbitmap_queue_free(&tags->bitmap_tags);
	360	free_tags:
	361	kfree(tags);
	362	return NULL;
	363	}
	364
	365	struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
	366	unsigned int reserved_tags,
	367	int node, int alloc_policy)
	368	{
	369	struct blk_mq_tags *tags;
	370
	371	if (total_tags > BLK_MQ_TAG_MAX) {
	372	pr_err("blk-mq: tag depth too large\n");
	373	return NULL;
	374	}
	375
	376	tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node);
	377	if (!tags)
	378	return NULL;
	379
	380	tags->nr_tags = total_tags;
	381	tags->nr_reserved_tags = reserved_tags;
	382
	383	return blk_mq_init_bitmap_tags(tags, node, alloc_policy);
	384	}
	385
	386	void blk_mq_free_tags(struct blk_mq_tags *tags)
	387	{
	388	sbitmap_queue_free(&tags->bitmap_tags);
	389	sbitmap_queue_free(&tags->breserved_tags);
	390	kfree(tags);
	391	}
	392
	393	int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
	394	struct blk_mq_tags **tagsptr, unsigned int tdepth,
	395	bool can_grow)
	396	{
	397	struct blk_mq_tags tags = tagsptr;
	398
	399	if (tdepth <= tags->nr_reserved_tags)
	400	return -EINVAL;
	401
	402	tdepth -= tags->nr_reserved_tags;
	403
	404	/*
	405	* If we are allowed to grow beyond the original size, allocate
	406	* a new set of tags before freeing the old one.
	407	*/
	408	if (tdepth > tags->nr_tags) {
	409	struct blk_mq_tag_set *set = hctx->queue->tag_set;
	410	struct blk_mq_tags *new;
	411	bool ret;
	412
	413	if (!can_grow)
	414	return -EINVAL;
	415
	416	/*
	417	* We need some sort of upper limit, set it high enough that
	418	* no valid use cases should require more.
	419	*/
	420	if (tdepth > 16 * BLKDEV_MAX_RQ)
	421	return -EINVAL;
	422
	423	new = blk_mq_alloc_rq_map(set, hctx->queue_num, tdepth, 0);
	424	if (!new)
	425	return -ENOMEM;
	426	ret = blk_mq_alloc_rqs(set, new, hctx->queue_num, tdepth);
	427	if (ret) {
	428	blk_mq_free_rq_map(new);
	429	return -ENOMEM;
	430	}
	431
	432	blk_mq_free_rqs(set, *tagsptr, hctx->queue_num);
	433	blk_mq_free_rq_map(*tagsptr);
	434	*tagsptr = new;
	435	} else {
	436	/*
	437	* Don't need (or can't) update reserved tags here, they
	438	* remain static and should never need resizing.
	439	*/
	440	sbitmap_queue_resize(&tags->bitmap_tags, tdepth);
	441	}
	442
	443	return 0;
	444	}
	445
	446	/**
	447	* blk_mq_unique_tag() - return a tag that is unique queue-wide
	448	* @rq: request for which to compute a unique tag
	449	*
	450	* The tag field in struct request is unique per hardware queue but not over
	451	* all hardware queues. Hence this function that returns a tag with the
	452	* hardware context index in the upper bits and the per hardware queue tag in
	453	* the lower bits.
	454	*
	455	* Note: When called for a request that is queued on a non-multiqueue request
	456	* queue, the hardware context index is set to zero.
	457	*/
	458	u32 blk_mq_unique_tag(struct request *rq)
	459	{
	460	struct request_queue *q = rq->q;
	461	struct blk_mq_hw_ctx *hctx;
	462	int hwq = 0;
	463
	464	if (q->mq_ops) {
	465	hctx = blk_mq_map_queue(q, rq->mq_ctx->cpu);
	466	hwq = hctx->queue_num;
	467	}
	468
	469	return (hwq << BLK_MQ_UNIQUE_TAG_BITS) \|
	470	(rq->tag & BLK_MQ_UNIQUE_TAG_MASK);
	471	}
	472	EXPORT_SYMBOL(blk_mq_unique_tag);