| 1 | /* |
| 2 | * blk-mq scheduling framework |
| 3 | * |
| 4 | * Copyright (C) 2016 Jens Axboe |
| 5 | */ |
| 6 | #include <linux/kernel.h> |
| 7 | #include <linux/module.h> |
| 8 | #include <linux/blk-mq.h> |
| 9 | |
| 10 | #include <trace/events/block.h> |
| 11 | |
| 12 | #include "blk.h" |
| 13 | #include "blk-mq.h" |
| 14 | #include "blk-mq-debugfs.h" |
| 15 | #include "blk-mq-sched.h" |
| 16 | #include "blk-mq-tag.h" |
| 17 | #include "blk-wbt.h" |
| 18 | |
| 19 | void blk_mq_sched_free_hctx_data(struct request_queue *q, |
| 20 | void (*exit)(struct blk_mq_hw_ctx *)) |
| 21 | { |
| 22 | struct blk_mq_hw_ctx *hctx; |
| 23 | int i; |
| 24 | |
| 25 | queue_for_each_hw_ctx(q, hctx, i) { |
| 26 | if (exit && hctx->sched_data) |
| 27 | exit(hctx); |
| 28 | kfree(hctx->sched_data); |
| 29 | hctx->sched_data = NULL; |
| 30 | } |
| 31 | } |
| 32 | EXPORT_SYMBOL_GPL(blk_mq_sched_free_hctx_data); |
| 33 | |
| 34 | void blk_mq_sched_assign_ioc(struct request *rq, struct bio *bio) |
| 35 | { |
| 36 | struct request_queue *q = rq->q; |
| 37 | struct io_context *ioc = rq_ioc(bio); |
| 38 | struct io_cq *icq; |
| 39 | |
| 40 | spin_lock_irq(q->queue_lock); |
| 41 | icq = ioc_lookup_icq(ioc, q); |
| 42 | spin_unlock_irq(q->queue_lock); |
| 43 | |
| 44 | if (!icq) { |
| 45 | icq = ioc_create_icq(ioc, q, GFP_ATOMIC); |
| 46 | if (!icq) |
| 47 | return; |
| 48 | } |
| 49 | get_io_context(icq->ioc); |
| 50 | rq->elv.icq = icq; |
| 51 | } |
| 52 | |
| 53 | /* |
| 54 | * Mark a hardware queue as needing a restart. For shared queues, maintain |
| 55 | * a count of how many hardware queues are marked for restart. |
| 56 | */ |
| 57 | static void blk_mq_sched_mark_restart_hctx(struct blk_mq_hw_ctx *hctx) |
| 58 | { |
| 59 | if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) |
| 60 | return; |
| 61 | |
| 62 | set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); |
| 63 | } |
| 64 | |
| 65 | void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx) |
| 66 | { |
| 67 | if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) |
| 68 | return; |
| 69 | clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); |
| 70 | |
| 71 | blk_mq_run_hw_queue(hctx, true); |
| 72 | } |
| 73 | |
| 74 | /* |
| 75 | * Only SCSI implements .get_budget and .put_budget, and SCSI restarts |
| 76 | * its queue by itself in its completion handler, so we don't need to |
| 77 | * restart queue if .get_budget() returns BLK_STS_NO_RESOURCE. |
| 78 | */ |
| 79 | static void blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx) |
| 80 | { |
| 81 | struct request_queue *q = hctx->queue; |
| 82 | struct elevator_queue *e = q->elevator; |
| 83 | LIST_HEAD(rq_list); |
| 84 | |
| 85 | do { |
| 86 | struct request *rq; |
| 87 | |
| 88 | if (e->type->ops.mq.has_work && |
| 89 | !e->type->ops.mq.has_work(hctx)) |
| 90 | break; |
| 91 | |
| 92 | if (!blk_mq_get_dispatch_budget(hctx)) |
| 93 | break; |
| 94 | |
| 95 | rq = e->type->ops.mq.dispatch_request(hctx); |
| 96 | if (!rq) { |
| 97 | blk_mq_put_dispatch_budget(hctx); |
| 98 | break; |
| 99 | } |
| 100 | |
| 101 | /* |
| 102 | * Now this rq owns the budget which has to be released |
| 103 | * if this rq won't be queued to driver via .queue_rq() |
| 104 | * in blk_mq_dispatch_rq_list(). |
| 105 | */ |
| 106 | list_add(&rq->queuelist, &rq_list); |
| 107 | } while (blk_mq_dispatch_rq_list(q, &rq_list, true)); |
| 108 | } |
| 109 | |
| 110 | static struct blk_mq_ctx *blk_mq_next_ctx(struct blk_mq_hw_ctx *hctx, |
| 111 | struct blk_mq_ctx *ctx) |
| 112 | { |
| 113 | unsigned idx = ctx->index_hw; |
| 114 | |
| 115 | if (++idx == hctx->nr_ctx) |
| 116 | idx = 0; |
| 117 | |
| 118 | return hctx->ctxs[idx]; |
| 119 | } |
| 120 | |
| 121 | /* |
| 122 | * Only SCSI implements .get_budget and .put_budget, and SCSI restarts |
| 123 | * its queue by itself in its completion handler, so we don't need to |
| 124 | * restart queue if .get_budget() returns BLK_STS_NO_RESOURCE. |
| 125 | */ |
| 126 | static void blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx) |
| 127 | { |
| 128 | struct request_queue *q = hctx->queue; |
| 129 | LIST_HEAD(rq_list); |
| 130 | struct blk_mq_ctx *ctx = READ_ONCE(hctx->dispatch_from); |
| 131 | |
| 132 | do { |
| 133 | struct request *rq; |
| 134 | |
| 135 | if (!sbitmap_any_bit_set(&hctx->ctx_map)) |
| 136 | break; |
| 137 | |
| 138 | if (!blk_mq_get_dispatch_budget(hctx)) |
| 139 | break; |
| 140 | |
| 141 | rq = blk_mq_dequeue_from_ctx(hctx, ctx); |
| 142 | if (!rq) { |
| 143 | blk_mq_put_dispatch_budget(hctx); |
| 144 | break; |
| 145 | } |
| 146 | |
| 147 | /* |
| 148 | * Now this rq owns the budget which has to be released |
| 149 | * if this rq won't be queued to driver via .queue_rq() |
| 150 | * in blk_mq_dispatch_rq_list(). |
| 151 | */ |
| 152 | list_add(&rq->queuelist, &rq_list); |
| 153 | |
| 154 | /* round robin for fair dispatch */ |
| 155 | ctx = blk_mq_next_ctx(hctx, rq->mq_ctx); |
| 156 | |
| 157 | } while (blk_mq_dispatch_rq_list(q, &rq_list, true)); |
| 158 | |
| 159 | WRITE_ONCE(hctx->dispatch_from, ctx); |
| 160 | } |
| 161 | |
| 162 | void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx) |
| 163 | { |
| 164 | struct request_queue *q = hctx->queue; |
| 165 | struct elevator_queue *e = q->elevator; |
| 166 | const bool has_sched_dispatch = e && e->type->ops.mq.dispatch_request; |
| 167 | LIST_HEAD(rq_list); |
| 168 | |
| 169 | /* RCU or SRCU read lock is needed before checking quiesced flag */ |
| 170 | if (unlikely(blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q))) |
| 171 | return; |
| 172 | |
| 173 | hctx->run++; |
| 174 | |
| 175 | /* |
| 176 | * If we have previous entries on our dispatch list, grab them first for |
| 177 | * more fair dispatch. |
| 178 | */ |
| 179 | if (!list_empty_careful(&hctx->dispatch)) { |
| 180 | spin_lock(&hctx->lock); |
| 181 | if (!list_empty(&hctx->dispatch)) |
| 182 | list_splice_init(&hctx->dispatch, &rq_list); |
| 183 | spin_unlock(&hctx->lock); |
| 184 | } |
| 185 | |
| 186 | /* |
| 187 | * Only ask the scheduler for requests, if we didn't have residual |
| 188 | * requests from the dispatch list. This is to avoid the case where |
| 189 | * we only ever dispatch a fraction of the requests available because |
| 190 | * of low device queue depth. Once we pull requests out of the IO |
| 191 | * scheduler, we can no longer merge or sort them. So it's best to |
| 192 | * leave them there for as long as we can. Mark the hw queue as |
| 193 | * needing a restart in that case. |
| 194 | * |
| 195 | * We want to dispatch from the scheduler if there was nothing |
| 196 | * on the dispatch list or we were able to dispatch from the |
| 197 | * dispatch list. |
| 198 | */ |
| 199 | if (!list_empty(&rq_list)) { |
| 200 | blk_mq_sched_mark_restart_hctx(hctx); |
| 201 | if (blk_mq_dispatch_rq_list(q, &rq_list, false)) { |
| 202 | if (has_sched_dispatch) |
| 203 | blk_mq_do_dispatch_sched(hctx); |
| 204 | else |
| 205 | blk_mq_do_dispatch_ctx(hctx); |
| 206 | } |
| 207 | } else if (has_sched_dispatch) { |
| 208 | blk_mq_do_dispatch_sched(hctx); |
| 209 | } else if (q->mq_ops->get_budget) { |
| 210 | /* |
| 211 | * If we need to get budget before queuing request, we |
| 212 | * dequeue request one by one from sw queue for avoiding |
| 213 | * to mess up I/O merge when dispatch runs out of resource. |
| 214 | * |
| 215 | * TODO: get more budgets, and dequeue more requests in |
| 216 | * one time. |
| 217 | */ |
| 218 | blk_mq_do_dispatch_ctx(hctx); |
| 219 | } else { |
| 220 | blk_mq_flush_busy_ctxs(hctx, &rq_list); |
| 221 | blk_mq_dispatch_rq_list(q, &rq_list, false); |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio, |
| 226 | struct request **merged_request) |
| 227 | { |
| 228 | struct request *rq; |
| 229 | |
| 230 | switch (elv_merge(q, &rq, bio)) { |
| 231 | case ELEVATOR_BACK_MERGE: |
| 232 | if (!blk_mq_sched_allow_merge(q, rq, bio)) |
| 233 | return false; |
| 234 | if (!bio_attempt_back_merge(q, rq, bio)) |
| 235 | return false; |
| 236 | *merged_request = attempt_back_merge(q, rq); |
| 237 | if (!*merged_request) |
| 238 | elv_merged_request(q, rq, ELEVATOR_BACK_MERGE); |
| 239 | return true; |
| 240 | case ELEVATOR_FRONT_MERGE: |
| 241 | if (!blk_mq_sched_allow_merge(q, rq, bio)) |
| 242 | return false; |
| 243 | if (!bio_attempt_front_merge(q, rq, bio)) |
| 244 | return false; |
| 245 | *merged_request = attempt_front_merge(q, rq); |
| 246 | if (!*merged_request) |
| 247 | elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE); |
| 248 | return true; |
| 249 | case ELEVATOR_DISCARD_MERGE: |
| 250 | return bio_attempt_discard_merge(q, rq, bio); |
| 251 | default: |
| 252 | return false; |
| 253 | } |
| 254 | } |
| 255 | EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge); |
| 256 | |
| 257 | /* |
| 258 | * Iterate list of requests and see if we can merge this bio with any |
| 259 | * of them. |
| 260 | */ |
| 261 | bool blk_mq_bio_list_merge(struct request_queue *q, struct list_head *list, |
| 262 | struct bio *bio) |
| 263 | { |
| 264 | struct request *rq; |
| 265 | int checked = 8; |
| 266 | |
| 267 | list_for_each_entry_reverse(rq, list, queuelist) { |
| 268 | bool merged = false; |
| 269 | |
| 270 | if (!checked--) |
| 271 | break; |
| 272 | |
| 273 | if (!blk_rq_merge_ok(rq, bio)) |
| 274 | continue; |
| 275 | |
| 276 | switch (blk_try_merge(rq, bio)) { |
| 277 | case ELEVATOR_BACK_MERGE: |
| 278 | if (blk_mq_sched_allow_merge(q, rq, bio)) |
| 279 | merged = bio_attempt_back_merge(q, rq, bio); |
| 280 | break; |
| 281 | case ELEVATOR_FRONT_MERGE: |
| 282 | if (blk_mq_sched_allow_merge(q, rq, bio)) |
| 283 | merged = bio_attempt_front_merge(q, rq, bio); |
| 284 | break; |
| 285 | case ELEVATOR_DISCARD_MERGE: |
| 286 | merged = bio_attempt_discard_merge(q, rq, bio); |
| 287 | break; |
| 288 | default: |
| 289 | continue; |
| 290 | } |
| 291 | |
| 292 | return merged; |
| 293 | } |
| 294 | |
| 295 | return false; |
| 296 | } |
| 297 | EXPORT_SYMBOL_GPL(blk_mq_bio_list_merge); |
| 298 | |
| 299 | /* |
| 300 | * Reverse check our software queue for entries that we could potentially |
| 301 | * merge with. Currently includes a hand-wavy stop count of 8, to not spend |
| 302 | * too much time checking for merges. |
| 303 | */ |
| 304 | static bool blk_mq_attempt_merge(struct request_queue *q, |
| 305 | struct blk_mq_ctx *ctx, struct bio *bio) |
| 306 | { |
| 307 | lockdep_assert_held(&ctx->lock); |
| 308 | |
| 309 | if (blk_mq_bio_list_merge(q, &ctx->rq_list, bio)) { |
| 310 | ctx->rq_merged++; |
| 311 | return true; |
| 312 | } |
| 313 | |
| 314 | return false; |
| 315 | } |
| 316 | |
| 317 | bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio) |
| 318 | { |
| 319 | struct elevator_queue *e = q->elevator; |
| 320 | struct blk_mq_ctx *ctx = blk_mq_get_ctx(q); |
| 321 | struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu); |
| 322 | bool ret = false; |
| 323 | |
| 324 | if (e && e->type->ops.mq.bio_merge) { |
| 325 | blk_mq_put_ctx(ctx); |
| 326 | return e->type->ops.mq.bio_merge(hctx, bio); |
| 327 | } |
| 328 | |
| 329 | if ((hctx->flags & BLK_MQ_F_SHOULD_MERGE) && |
| 330 | !list_empty_careful(&ctx->rq_list)) { |
| 331 | /* default per sw-queue merge */ |
| 332 | spin_lock(&ctx->lock); |
| 333 | ret = blk_mq_attempt_merge(q, ctx, bio); |
| 334 | spin_unlock(&ctx->lock); |
| 335 | } |
| 336 | |
| 337 | blk_mq_put_ctx(ctx); |
| 338 | return ret; |
| 339 | } |
| 340 | |
| 341 | bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq) |
| 342 | { |
| 343 | return rq_mergeable(rq) && elv_attempt_insert_merge(q, rq); |
| 344 | } |
| 345 | EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge); |
| 346 | |
| 347 | void blk_mq_sched_request_inserted(struct request *rq) |
| 348 | { |
| 349 | trace_block_rq_insert(rq->q, rq); |
| 350 | } |
| 351 | EXPORT_SYMBOL_GPL(blk_mq_sched_request_inserted); |
| 352 | |
| 353 | static bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx, |
| 354 | bool has_sched, |
| 355 | struct request *rq) |
| 356 | { |
| 357 | /* dispatch flush rq directly */ |
| 358 | if (rq->rq_flags & RQF_FLUSH_SEQ) { |
| 359 | spin_lock(&hctx->lock); |
| 360 | list_add(&rq->queuelist, &hctx->dispatch); |
| 361 | spin_unlock(&hctx->lock); |
| 362 | return true; |
| 363 | } |
| 364 | |
| 365 | if (has_sched) |
| 366 | rq->rq_flags |= RQF_SORTED; |
| 367 | |
| 368 | return false; |
| 369 | } |
| 370 | |
| 371 | void blk_mq_sched_insert_request(struct request *rq, bool at_head, |
| 372 | bool run_queue, bool async) |
| 373 | { |
| 374 | struct request_queue *q = rq->q; |
| 375 | struct elevator_queue *e = q->elevator; |
| 376 | struct blk_mq_ctx *ctx = rq->mq_ctx; |
| 377 | struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu); |
| 378 | |
| 379 | /* flush rq in flush machinery need to be dispatched directly */ |
| 380 | if (!(rq->rq_flags & RQF_FLUSH_SEQ) && op_is_flush(rq->cmd_flags)) { |
| 381 | blk_insert_flush(rq); |
| 382 | goto run; |
| 383 | } |
| 384 | |
| 385 | WARN_ON(e && (rq->tag != -1)); |
| 386 | |
| 387 | if (blk_mq_sched_bypass_insert(hctx, !!e, rq)) |
| 388 | goto run; |
| 389 | |
| 390 | if (e && e->type->ops.mq.insert_requests) { |
| 391 | LIST_HEAD(list); |
| 392 | |
| 393 | list_add(&rq->queuelist, &list); |
| 394 | e->type->ops.mq.insert_requests(hctx, &list, at_head); |
| 395 | } else { |
| 396 | spin_lock(&ctx->lock); |
| 397 | __blk_mq_insert_request(hctx, rq, at_head); |
| 398 | spin_unlock(&ctx->lock); |
| 399 | } |
| 400 | |
| 401 | run: |
| 402 | if (run_queue) |
| 403 | blk_mq_run_hw_queue(hctx, async); |
| 404 | } |
| 405 | |
| 406 | void blk_mq_sched_insert_requests(struct request_queue *q, |
| 407 | struct blk_mq_ctx *ctx, |
| 408 | struct list_head *list, bool run_queue_async) |
| 409 | { |
| 410 | struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu); |
| 411 | struct elevator_queue *e = hctx->queue->elevator; |
| 412 | |
| 413 | if (e && e->type->ops.mq.insert_requests) |
| 414 | e->type->ops.mq.insert_requests(hctx, list, false); |
| 415 | else |
| 416 | blk_mq_insert_requests(hctx, ctx, list); |
| 417 | |
| 418 | blk_mq_run_hw_queue(hctx, run_queue_async); |
| 419 | } |
| 420 | |
| 421 | static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set, |
| 422 | struct blk_mq_hw_ctx *hctx, |
| 423 | unsigned int hctx_idx) |
| 424 | { |
| 425 | if (hctx->sched_tags) { |
| 426 | blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx); |
| 427 | blk_mq_free_rq_map(hctx->sched_tags); |
| 428 | hctx->sched_tags = NULL; |
| 429 | } |
| 430 | } |
| 431 | |
| 432 | static int blk_mq_sched_alloc_tags(struct request_queue *q, |
| 433 | struct blk_mq_hw_ctx *hctx, |
| 434 | unsigned int hctx_idx) |
| 435 | { |
| 436 | struct blk_mq_tag_set *set = q->tag_set; |
| 437 | int ret; |
| 438 | |
| 439 | hctx->sched_tags = blk_mq_alloc_rq_map(set, hctx_idx, q->nr_requests, |
| 440 | set->reserved_tags); |
| 441 | if (!hctx->sched_tags) |
| 442 | return -ENOMEM; |
| 443 | |
| 444 | ret = blk_mq_alloc_rqs(set, hctx->sched_tags, hctx_idx, q->nr_requests); |
| 445 | if (ret) |
| 446 | blk_mq_sched_free_tags(set, hctx, hctx_idx); |
| 447 | |
| 448 | return ret; |
| 449 | } |
| 450 | |
| 451 | static void blk_mq_sched_tags_teardown(struct request_queue *q) |
| 452 | { |
| 453 | struct blk_mq_tag_set *set = q->tag_set; |
| 454 | struct blk_mq_hw_ctx *hctx; |
| 455 | int i; |
| 456 | |
| 457 | queue_for_each_hw_ctx(q, hctx, i) |
| 458 | blk_mq_sched_free_tags(set, hctx, i); |
| 459 | } |
| 460 | |
| 461 | int blk_mq_sched_init_hctx(struct request_queue *q, struct blk_mq_hw_ctx *hctx, |
| 462 | unsigned int hctx_idx) |
| 463 | { |
| 464 | struct elevator_queue *e = q->elevator; |
| 465 | int ret; |
| 466 | |
| 467 | if (!e) |
| 468 | return 0; |
| 469 | |
| 470 | ret = blk_mq_sched_alloc_tags(q, hctx, hctx_idx); |
| 471 | if (ret) |
| 472 | return ret; |
| 473 | |
| 474 | if (e->type->ops.mq.init_hctx) { |
| 475 | ret = e->type->ops.mq.init_hctx(hctx, hctx_idx); |
| 476 | if (ret) { |
| 477 | blk_mq_sched_free_tags(q->tag_set, hctx, hctx_idx); |
| 478 | return ret; |
| 479 | } |
| 480 | } |
| 481 | |
| 482 | blk_mq_debugfs_register_sched_hctx(q, hctx); |
| 483 | |
| 484 | return 0; |
| 485 | } |
| 486 | |
| 487 | void blk_mq_sched_exit_hctx(struct request_queue *q, struct blk_mq_hw_ctx *hctx, |
| 488 | unsigned int hctx_idx) |
| 489 | { |
| 490 | struct elevator_queue *e = q->elevator; |
| 491 | |
| 492 | if (!e) |
| 493 | return; |
| 494 | |
| 495 | blk_mq_debugfs_unregister_sched_hctx(hctx); |
| 496 | |
| 497 | if (e->type->ops.mq.exit_hctx && hctx->sched_data) { |
| 498 | e->type->ops.mq.exit_hctx(hctx, hctx_idx); |
| 499 | hctx->sched_data = NULL; |
| 500 | } |
| 501 | |
| 502 | blk_mq_sched_free_tags(q->tag_set, hctx, hctx_idx); |
| 503 | } |
| 504 | |
| 505 | int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e) |
| 506 | { |
| 507 | struct blk_mq_hw_ctx *hctx; |
| 508 | struct elevator_queue *eq; |
| 509 | unsigned int i; |
| 510 | int ret; |
| 511 | |
| 512 | if (!e) { |
| 513 | q->elevator = NULL; |
| 514 | q->nr_requests = q->tag_set->queue_depth; |
| 515 | return 0; |
| 516 | } |
| 517 | |
| 518 | /* |
| 519 | * Default to double of smaller one between hw queue_depth and 128, |
| 520 | * since we don't split into sync/async like the old code did. |
| 521 | * Additionally, this is a per-hw queue depth. |
| 522 | */ |
| 523 | q->nr_requests = 2 * min_t(unsigned int, q->tag_set->queue_depth, |
| 524 | BLKDEV_MAX_RQ); |
| 525 | |
| 526 | queue_for_each_hw_ctx(q, hctx, i) { |
| 527 | ret = blk_mq_sched_alloc_tags(q, hctx, i); |
| 528 | if (ret) |
| 529 | goto err; |
| 530 | } |
| 531 | |
| 532 | ret = e->ops.mq.init_sched(q, e); |
| 533 | if (ret) |
| 534 | goto err; |
| 535 | |
| 536 | blk_mq_debugfs_register_sched(q); |
| 537 | |
| 538 | queue_for_each_hw_ctx(q, hctx, i) { |
| 539 | if (e->ops.mq.init_hctx) { |
| 540 | ret = e->ops.mq.init_hctx(hctx, i); |
| 541 | if (ret) { |
| 542 | eq = q->elevator; |
| 543 | blk_mq_exit_sched(q, eq); |
| 544 | kobject_put(&eq->kobj); |
| 545 | return ret; |
| 546 | } |
| 547 | } |
| 548 | blk_mq_debugfs_register_sched_hctx(q, hctx); |
| 549 | } |
| 550 | |
| 551 | return 0; |
| 552 | |
| 553 | err: |
| 554 | blk_mq_sched_tags_teardown(q); |
| 555 | q->elevator = NULL; |
| 556 | return ret; |
| 557 | } |
| 558 | |
| 559 | void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e) |
| 560 | { |
| 561 | struct blk_mq_hw_ctx *hctx; |
| 562 | unsigned int i; |
| 563 | |
| 564 | queue_for_each_hw_ctx(q, hctx, i) { |
| 565 | blk_mq_debugfs_unregister_sched_hctx(hctx); |
| 566 | if (e->type->ops.mq.exit_hctx && hctx->sched_data) { |
| 567 | e->type->ops.mq.exit_hctx(hctx, i); |
| 568 | hctx->sched_data = NULL; |
| 569 | } |
| 570 | } |
| 571 | blk_mq_debugfs_unregister_sched(q); |
| 572 | if (e->type->ops.mq.exit_sched) |
| 573 | e->type->ops.mq.exit_sched(e); |
| 574 | blk_mq_sched_tags_teardown(q); |
| 575 | q->elevator = NULL; |
| 576 | } |