block: store elevator state in request

Add an rq private RQF_ELV flag, which tells the block layer that this
request was initialized on a queue that has an IO scheduler attached.
This allows for faster checking in the fast path, rather than having to
deference rq->q later on.

Elevator switching does full quiesce of the queue before detaching an
IO scheduler, so it's safe to cache this in the request itself.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h
index fe252278ed9a564e0dfd864e89650fd385ff6885..98836106b25fc27ca2731438c207f458a7b30570 100644 (file)
--- a/block/blk-mq-sched.h
+++ b/block/blk-mq-sched.h
@@ -56,29 +56,34 @@ static inline bool
 blk_mq_sched_allow_merge(struct request_queue *q, struct request *rq,
                         struct bio *bio)
 {
-       struct elevator_queue *e = q->elevator;
-
-       if (e && e->type->ops.allow_merge)
-               return e->type->ops.allow_merge(q, rq, bio);
+       if (rq->rq_flags & RQF_ELV) {
+               struct elevator_queue *e = q->elevator;
 
+               if (e->type->ops.allow_merge)
+                       return e->type->ops.allow_merge(q, rq, bio);
+       }
        return true;
 }
 
 static inline void blk_mq_sched_completed_request(struct request *rq, u64 now)
 {
-       struct elevator_queue *e = rq->q->elevator;
+       if (rq->rq_flags & RQF_ELV) {
+               struct elevator_queue *e = rq->q->elevator;
 
-       if (e && e->type->ops.completed_request)
-               e->type->ops.completed_request(rq, now);
+               if (e->type->ops.completed_request)
+                       e->type->ops.completed_request(rq, now);
+       }
 }
 
 static inline void blk_mq_sched_requeue_request(struct request *rq)
 {
-       struct request_queue *q = rq->q;
-       struct elevator_queue *e = q->elevator;
+       if (rq->rq_flags & RQF_ELV) {
+               struct request_queue *q = rq->q;
+               struct elevator_queue *e = q->elevator;
 
-       if ((rq->rq_flags & RQF_ELVPRIV) && e && e->type->ops.requeue_request)
-               e->type->ops.requeue_request(rq);
+               if ((rq->rq_flags & RQF_ELVPRIV) && e->type->ops.requeue_request)
+                       e->type->ops.requeue_request(rq);
+       }
 }
 
 static inline bool blk_mq_sched_has_work(struct blk_mq_hw_ctx *hctx)

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 9cff9e8eada40a27639b11818378de24029fc65a..28eb1f3c6f76f955f34c61f948a4c4f1a9b48f27 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -306,7 +306,7 @@ void blk_mq_wake_waiters(struct request_queue *q)
  */
 static inline bool blk_mq_need_time_stamp(struct request *rq)
 {
-       return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS)) || rq->q->elevator;
+       return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_ELV));
 }
 
 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
@@ -316,9 +316,11 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
        struct request *rq = tags->static_rqs[tag];
 
        if (data->q->elevator) {
+               rq->rq_flags = RQF_ELV;
                rq->tag = BLK_MQ_NO_TAG;
                rq->internal_tag = tag;
        } else {
+               rq->rq_flags = 0;
                rq->tag = tag;
                rq->internal_tag = BLK_MQ_NO_TAG;
        }
@@ -327,7 +329,6 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
        rq->q = data->q;
        rq->mq_ctx = data->ctx;
        rq->mq_hctx = data->hctx;
-       rq->rq_flags = 0;
        rq->cmd_flags = data->cmd_flags;
        if (data->flags & BLK_MQ_REQ_PM)
                rq->rq_flags |= RQF_PM;
@@ -363,11 +364,11 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
        data->ctx->rq_dispatched[op_is_sync(data->cmd_flags)]++;
        refcount_set(&rq->ref, 1);
 
-       if (!op_is_flush(data->cmd_flags)) {
+       if (!op_is_flush(data->cmd_flags) && (rq->rq_flags & RQF_ELV)) {
                struct elevator_queue *e = data->q->elevator;
 
                rq->elv.icq = NULL;
-               if (e && e->type->ops.prepare_request) {
+               if (e->type->ops.prepare_request) {
                        if (e->type->icq_cache)
                                blk_mq_sched_assign_ioc(rq);
 
@@ -588,12 +589,13 @@ static void __blk_mq_free_request(struct request *rq)
 void blk_mq_free_request(struct request *rq)
 {
        struct request_queue *q = rq->q;
-       struct elevator_queue *e = q->elevator;
        struct blk_mq_ctx *ctx = rq->mq_ctx;
        struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
 
-       if (rq->rq_flags & RQF_ELVPRIV) {
-               if (e && e->type->ops.finish_request)
+       if (rq->rq_flags & (RQF_ELVPRIV | RQF_ELV)) {
+               struct elevator_queue *e = q->elevator;
+
+               if (e->type->ops.finish_request)
                        e->type->ops.finish_request(rq);
                if (rq->elv.icq) {
                        put_io_context(rq->elv.icq->ioc);
@@ -2254,7 +2256,7 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
                goto insert;
        }
 
-       if (q->elevator && !bypass_insert)
+       if ((rq->rq_flags & RQF_ELV) && !bypass_insert)
                goto insert;
 
        budget_token = blk_mq_get_dispatch_budget(q);
@@ -2492,7 +2494,7 @@ void blk_mq_submit_bio(struct bio *bio)
                }
 
                blk_add_rq_to_plug(plug, rq);
-       } else if (q->elevator) {
+       } else if (rq->rq_flags & RQF_ELV) {
                /* Insert the request at the IO scheduler queue */
                blk_mq_sched_insert_request(rq, false, true, true);
        } else if (plug && !blk_queue_nomerges(q)) {

diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 8ca9728cc7f218140c0bd88b7eb0ee40b2d1fce2..95c3bd3a008e5bcefbd6b1955b1404094d6111aa 100644 (file)
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -55,6 +55,8 @@ typedef __u32 __bitwise req_flags_t;
 #define RQF_MQ_POLL_SLEPT      ((__force req_flags_t)(1 << 20))
 /* ->timeout has been called, don't expire again */
 #define RQF_TIMED_OUT          ((__force req_flags_t)(1 << 21))
+/* queue has elevator attached */
+#define RQF_ELV                        ((__force req_flags_t)(1 << 22))
 
 /* flags that prevent us from merging requests: */
 #define RQF_NOMERGE_FLAGS \