[linux-block.git] / block / blk-mq-sched.h

#ifndef BLK_MQ_SCHED_H
#define BLK_MQ_SCHED_H

#include "blk-mq.h"
#include "blk-mq-tag.h"

int blk_mq_sched_init_hctx_data(struct request_queue *q, size_t size,
				int (*init)(struct blk_mq_hw_ctx *),
				void (*exit)(struct blk_mq_hw_ctx *));

void blk_mq_sched_free_hctx_data(struct request_queue *q,
				 void (*exit)(struct blk_mq_hw_ctx *));

struct request *blk_mq_sched_get_request(struct request_queue *q, struct bio *bio, unsigned int op, struct blk_mq_alloc_data *data);
void blk_mq_sched_put_request(struct request *rq);

void blk_mq_sched_request_inserted(struct request *rq);
bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx, struct request *rq);
bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio);
bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio);
bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq);
void blk_mq_sched_restart_queues(struct blk_mq_hw_ctx *hctx);

void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx);
void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx,
			struct list_head *rq_list,
			struct request *(*get_rq)(struct blk_mq_hw_ctx *));

int blk_mq_sched_setup(struct request_queue *q);
void blk_mq_sched_teardown(struct request_queue *q);

int blk_mq_sched_init(struct request_queue *q);

static inline bool
blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio)
{
	struct elevator_queue *e = q->elevator;

	if (!e || blk_queue_nomerges(q) || !bio_mergeable(bio))
		return false;

	return __blk_mq_sched_bio_merge(q, bio);
}

static inline int blk_mq_sched_get_rq_priv(struct request_queue *q,
					   struct request *rq)
{
	struct elevator_queue *e = q->elevator;

	if (e && e->type->ops.mq.get_rq_priv)
		return e->type->ops.mq.get_rq_priv(q, rq);

	return 0;
}

static inline void blk_mq_sched_put_rq_priv(struct request_queue *q,
					    struct request *rq)
{
	struct elevator_queue *e = q->elevator;

	if (e && e->type->ops.mq.put_rq_priv)
		e->type->ops.mq.put_rq_priv(q, rq);
}

static inline void
blk_mq_sched_insert_request(struct request *rq, bool at_head, bool run_queue,
			    bool async)
{
	struct request_queue *q = rq->q;
	struct elevator_queue *e = q->elevator;
	struct blk_mq_ctx *ctx = rq->mq_ctx;
	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);

	if (e && e->type->ops.mq.insert_requests) {
		LIST_HEAD(list);

		list_add(&rq->queuelist, &list);
		e->type->ops.mq.insert_requests(hctx, &list, at_head);
	} else {
		spin_lock(&ctx->lock);
		__blk_mq_insert_request(hctx, rq, at_head);
		spin_unlock(&ctx->lock);
	}

	if (run_queue)
		blk_mq_run_hw_queue(hctx, async);
}

static inline void
blk_mq_sched_insert_requests(struct request_queue *q, struct blk_mq_ctx *ctx,
			     struct list_head *list, bool run_queue_async)
{
	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	struct elevator_queue *e = hctx->queue->elevator;

	if (e && e->type->ops.mq.insert_requests)
		e->type->ops.mq.insert_requests(hctx, list, false);
	else
		blk_mq_insert_requests(hctx, ctx, list);

	blk_mq_run_hw_queue(hctx, run_queue_async);
}

static inline bool
blk_mq_sched_allow_merge(struct request_queue *q, struct request *rq,
			 struct bio *bio)
{
	struct elevator_queue *e = q->elevator;

	if (e && e->type->ops.mq.allow_merge)
		return e->type->ops.mq.allow_merge(q, rq, bio);

	return true;
}

static inline void
blk_mq_sched_completed_request(struct blk_mq_hw_ctx *hctx, struct request *rq)
{
	struct elevator_queue *e = hctx->queue->elevator;

	if (e && e->type->ops.mq.completed_request)
		e->type->ops.mq.completed_request(hctx, rq);

	BUG_ON(rq->internal_tag == -1);

	blk_mq_put_tag(hctx, hctx->sched_tags, rq->mq_ctx, rq->internal_tag);
}

static inline void blk_mq_sched_started_request(struct request *rq)
{
	struct request_queue *q = rq->q;
	struct elevator_queue *e = q->elevator;

	if (e && e->type->ops.mq.started_request)
		e->type->ops.mq.started_request(rq);
}

static inline void blk_mq_sched_requeue_request(struct request *rq)
{
	struct request_queue *q = rq->q;
	struct elevator_queue *e = q->elevator;

	if (e && e->type->ops.mq.requeue_request)
		e->type->ops.mq.requeue_request(rq);
}

static inline bool blk_mq_sched_has_work(struct blk_mq_hw_ctx *hctx)
{
	struct elevator_queue *e = hctx->queue->elevator;

	if (e && e->type->ops.mq.has_work)
		return e->type->ops.mq.has_work(hctx);

	return false;
}

static inline void blk_mq_sched_mark_restart(struct blk_mq_hw_ctx *hctx)
{
	if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) {
		set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
		if (hctx->flags & BLK_MQ_F_TAG_SHARED) {
			struct request_queue *q = hctx->queue;

			if (!test_bit(QUEUE_FLAG_RESTART, &q->queue_flags))
				set_bit(QUEUE_FLAG_RESTART, &q->queue_flags);
		}
	}
}

static inline bool blk_mq_sched_needs_restart(struct blk_mq_hw_ctx *hctx)
{
	return test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
}

#endif
Commit	Line	Data
bd166ef1 JA	1	#ifndef BLK_MQ_SCHED_H
	2	#define BLK_MQ_SCHED_H
	3
	4	#include "blk-mq.h"
	5	#include "blk-mq-tag.h"
	6
	7	int blk_mq_sched_init_hctx_data(struct request_queue *q, size_t size,
	8	int (init)(struct blk_mq_hw_ctx ),
	9	void (exit)(struct blk_mq_hw_ctx ));
	10
	11	void blk_mq_sched_free_hctx_data(struct request_queue *q,
	12	void (exit)(struct blk_mq_hw_ctx ));
	13
	14	struct request blk_mq_sched_get_request(struct request_queue q, struct bio bio, unsigned int op, struct blk_mq_alloc_data data);
	15	void blk_mq_sched_put_request(struct request *rq);
	16
	17	void blk_mq_sched_request_inserted(struct request *rq);
	18	bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx hctx, struct request rq);
	19	bool blk_mq_sched_try_merge(struct request_queue q, struct bio bio);
	20	bool __blk_mq_sched_bio_merge(struct request_queue q, struct bio bio);
	21	bool blk_mq_sched_try_insert_merge(struct request_queue q, struct request rq);
50e1dab8	22	void blk_mq_sched_restart_queues(struct blk_mq_hw_ctx *hctx);
bd166ef1 JA	23
	24	void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx);
	25	void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx,
	26	struct list_head *rq_list,
	27	struct request (get_rq)(struct blk_mq_hw_ctx *));
	28
	29	int blk_mq_sched_setup(struct request_queue *q);
	30	void blk_mq_sched_teardown(struct request_queue *q);
	31
d3484991 JA	32	int blk_mq_sched_init(struct request_queue *q);
d3484991 JA	33
bd166ef1 JA	34	static inline bool
	35	blk_mq_sched_bio_merge(struct request_queue q, struct bio bio)
	36	{
	37	struct elevator_queue *e = q->elevator;
	38
	39	if (!e \|\| blk_queue_nomerges(q) \|\| !bio_mergeable(bio))
	40	return false;
	41
	42	return __blk_mq_sched_bio_merge(q, bio);
	43	}
	44
	45	static inline int blk_mq_sched_get_rq_priv(struct request_queue *q,
	46	struct request *rq)
	47	{
	48	struct elevator_queue *e = q->elevator;
	49
	50	if (e && e->type->ops.mq.get_rq_priv)
	51	return e->type->ops.mq.get_rq_priv(q, rq);
	52
	53	return 0;
	54	}
	55
	56	static inline void blk_mq_sched_put_rq_priv(struct request_queue *q,
	57	struct request *rq)
	58	{
	59	struct elevator_queue *e = q->elevator;
	60
	61	if (e && e->type->ops.mq.put_rq_priv)
	62	e->type->ops.mq.put_rq_priv(q, rq);
	63	}
	64
	65	static inline void
	66	blk_mq_sched_insert_request(struct request *rq, bool at_head, bool run_queue,
	67	bool async)
	68	{
	69	struct request_queue *q = rq->q;
	70	struct elevator_queue *e = q->elevator;
	71	struct blk_mq_ctx *ctx = rq->mq_ctx;
	72	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	73
	74	if (e && e->type->ops.mq.insert_requests) {
	75	LIST_HEAD(list);
	76
	77	list_add(&rq->queuelist, &list);
	78	e->type->ops.mq.insert_requests(hctx, &list, at_head);
	79	} else {
	80	spin_lock(&ctx->lock);
	81	__blk_mq_insert_request(hctx, rq, at_head);
	82	spin_unlock(&ctx->lock);
	83	}
	84
	85	if (run_queue)
	86	blk_mq_run_hw_queue(hctx, async);
	87	}
	88
	89	static inline void
	90	blk_mq_sched_insert_requests(struct request_queue q, struct blk_mq_ctx ctx,
	91	struct list_head *list, bool run_queue_async)
	92	{
	93	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	94	struct elevator_queue *e = hctx->queue->elevator;
	95
	96	if (e && e->type->ops.mq.insert_requests)
	97	e->type->ops.mq.insert_requests(hctx, list, false);
98	else
99	blk_mq_insert_requests(hctx, ctx, list);
100
101	blk_mq_run_hw_queue(hctx, run_queue_async);
102	}
103
104	static inline bool
105	blk_mq_sched_allow_merge(struct request_queue q, struct request rq,
106	struct bio *bio)
107	{
108	struct elevator_queue *e = q->elevator;
109
110	if (e && e->type->ops.mq.allow_merge)
111	return e->type->ops.mq.allow_merge(q, rq, bio);
112
113	return true;
114	}
115
116	static inline void
117	blk_mq_sched_completed_request(struct blk_mq_hw_ctx hctx, struct request rq)
118	{
119	struct elevator_queue *e = hctx->queue->elevator;
120
121	if (e && e->type->ops.mq.completed_request)
122	e->type->ops.mq.completed_request(hctx, rq);
123
124	BUG_ON(rq->internal_tag == -1);
125
126	blk_mq_put_tag(hctx, hctx->sched_tags, rq->mq_ctx, rq->internal_tag);
bd166ef1 JA	127	}
	128
	129	static inline void blk_mq_sched_started_request(struct request *rq)
	130	{
	131	struct request_queue *q = rq->q;
	132	struct elevator_queue *e = q->elevator;
	133
	134	if (e && e->type->ops.mq.started_request)
	135	e->type->ops.mq.started_request(rq);
	136	}
	137
	138	static inline void blk_mq_sched_requeue_request(struct request *rq)
	139	{
	140	struct request_queue *q = rq->q;
	141	struct elevator_queue *e = q->elevator;
	142
	143	if (e && e->type->ops.mq.requeue_request)
	144	e->type->ops.mq.requeue_request(rq);
	145	}
	146
	147	static inline bool blk_mq_sched_has_work(struct blk_mq_hw_ctx *hctx)
	148	{
	149	struct elevator_queue *e = hctx->queue->elevator;
	150
	151	if (e && e->type->ops.mq.has_work)
	152	return e->type->ops.mq.has_work(hctx);
	153
	154	return false;
	155	}
	156
	157	static inline void blk_mq_sched_mark_restart(struct blk_mq_hw_ctx *hctx)
	158	{
50e1dab8	159	if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) {
bd166ef1	160	set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
50e1dab8 JA	161	if (hctx->flags & BLK_MQ_F_TAG_SHARED) {
	162	struct request_queue *q = hctx->queue;
	163
	164	if (!test_bit(QUEUE_FLAG_RESTART, &q->queue_flags))
	165	set_bit(QUEUE_FLAG_RESTART, &q->queue_flags);
	166	}
	167	}
bd166ef1 JA	168	}
	169
	170	static inline bool blk_mq_sched_needs_restart(struct blk_mq_hw_ctx *hctx)
	171	{
	172	return test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
	173	}
	174
	175	#endif