block: remove useless caller argument to print_req_error()
[linux-block.git] / block / blk-mq.c
CommitLineData
3dcf60bc 1// SPDX-License-Identifier: GPL-2.0
75bb4625
JA
2/*
3 * Block multiqueue core code
4 *
5 * Copyright (C) 2013-2014 Jens Axboe
6 * Copyright (C) 2013-2014 Christoph Hellwig
7 */
320ae51f
JA
8#include <linux/kernel.h>
9#include <linux/module.h>
10#include <linux/backing-dev.h>
11#include <linux/bio.h>
12#include <linux/blkdev.h>
fe45e630 13#include <linux/blk-integrity.h>
f75782e4 14#include <linux/kmemleak.h>
320ae51f
JA
15#include <linux/mm.h>
16#include <linux/init.h>
17#include <linux/slab.h>
18#include <linux/workqueue.h>
19#include <linux/smp.h>
e41d12f5 20#include <linux/interrupt.h>
320ae51f
JA
21#include <linux/llist.h>
22#include <linux/list_sort.h>
23#include <linux/cpu.h>
24#include <linux/cache.h>
25#include <linux/sched/sysctl.h>
105ab3d8 26#include <linux/sched/topology.h>
174cd4b1 27#include <linux/sched/signal.h>
320ae51f 28#include <linux/delay.h>
aedcd72f 29#include <linux/crash_dump.h>
88c7b2b7 30#include <linux/prefetch.h>
a892c8d5 31#include <linux/blk-crypto.h>
320ae51f
JA
32
33#include <trace/events/block.h>
34
35#include <linux/blk-mq.h>
54d4e6ab 36#include <linux/t10-pi.h>
320ae51f
JA
37#include "blk.h"
38#include "blk-mq.h"
9c1051aa 39#include "blk-mq-debugfs.h"
320ae51f 40#include "blk-mq-tag.h"
986d413b 41#include "blk-pm.h"
cf43e6be 42#include "blk-stat.h"
bd166ef1 43#include "blk-mq-sched.h"
c1c80384 44#include "blk-rq-qos.h"
320ae51f 45
f9ab4918 46static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
c3077b5d 47
34dbad5d
OS
48static void blk_mq_poll_stats_start(struct request_queue *q);
49static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);
50
720b8ccc
SB
51static int blk_mq_poll_stats_bkt(const struct request *rq)
52{
3d244306 53 int ddir, sectors, bucket;
720b8ccc 54
99c749a4 55 ddir = rq_data_dir(rq);
3d244306 56 sectors = blk_rq_stats_sectors(rq);
720b8ccc 57
3d244306 58 bucket = ddir + 2 * ilog2(sectors);
720b8ccc
SB
59
60 if (bucket < 0)
61 return -1;
62 else if (bucket >= BLK_MQ_POLL_STATS_BKTS)
63 return ddir + BLK_MQ_POLL_STATS_BKTS - 2;
64
65 return bucket;
66}
67
3e08773c
CH
68#define BLK_QC_T_SHIFT 16
69#define BLK_QC_T_INTERNAL (1U << 31)
70
f70299f0
CH
71static inline struct blk_mq_hw_ctx *blk_qc_to_hctx(struct request_queue *q,
72 blk_qc_t qc)
73{
74 return q->queue_hw_ctx[(qc & ~BLK_QC_T_INTERNAL) >> BLK_QC_T_SHIFT];
75}
76
c6699d6f
CH
77static inline struct request *blk_qc_to_rq(struct blk_mq_hw_ctx *hctx,
78 blk_qc_t qc)
79{
efbabbe1
CH
80 unsigned int tag = qc & ((1U << BLK_QC_T_SHIFT) - 1);
81
82 if (qc & BLK_QC_T_INTERNAL)
83 return blk_mq_tag_to_rq(hctx->sched_tags, tag);
84 return blk_mq_tag_to_rq(hctx->tags, tag);
c6699d6f
CH
85}
86
3e08773c
CH
87static inline blk_qc_t blk_rq_to_qc(struct request *rq)
88{
89 return (rq->mq_hctx->queue_num << BLK_QC_T_SHIFT) |
90 (rq->tag != -1 ?
91 rq->tag : (rq->internal_tag | BLK_QC_T_INTERNAL));
92}
93
320ae51f 94/*
85fae294
YY
95 * Check if any of the ctx, dispatch list or elevator
96 * have pending work in this hardware queue.
320ae51f 97 */
79f720a7 98static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
320ae51f 99{
79f720a7
JA
100 return !list_empty_careful(&hctx->dispatch) ||
101 sbitmap_any_bit_set(&hctx->ctx_map) ||
bd166ef1 102 blk_mq_sched_has_work(hctx);
1429d7c9
JA
103}
104
320ae51f
JA
105/*
106 * Mark this ctx as having pending work in this hardware queue
107 */
108static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
109 struct blk_mq_ctx *ctx)
110{
f31967f0
JA
111 const int bit = ctx->index_hw[hctx->type];
112
113 if (!sbitmap_test_bit(&hctx->ctx_map, bit))
114 sbitmap_set_bit(&hctx->ctx_map, bit);
1429d7c9
JA
115}
116
117static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
118 struct blk_mq_ctx *ctx)
119{
f31967f0
JA
120 const int bit = ctx->index_hw[hctx->type];
121
122 sbitmap_clear_bit(&hctx->ctx_map, bit);
320ae51f
JA
123}
124
f299b7c7 125struct mq_inflight {
8446fe92 126 struct block_device *part;
a2e80f6f 127 unsigned int inflight[2];
f299b7c7
JA
128};
129
7baa8572 130static bool blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx,
f299b7c7
JA
131 struct request *rq, void *priv,
132 bool reserved)
133{
134 struct mq_inflight *mi = priv;
135
b0d97557
JX
136 if ((!mi->part->bd_partno || rq->part == mi->part) &&
137 blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
bb4e6b14 138 mi->inflight[rq_data_dir(rq)]++;
7baa8572
JA
139
140 return true;
f299b7c7
JA
141}
142
8446fe92
CH
143unsigned int blk_mq_in_flight(struct request_queue *q,
144 struct block_device *part)
f299b7c7 145{
a2e80f6f 146 struct mq_inflight mi = { .part = part };
f299b7c7 147
f299b7c7 148 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
e016b782 149
a2e80f6f 150 return mi.inflight[0] + mi.inflight[1];
bf0ddaba
OS
151}
152
8446fe92
CH
153void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part,
154 unsigned int inflight[2])
bf0ddaba 155{
a2e80f6f 156 struct mq_inflight mi = { .part = part };
bf0ddaba 157
bb4e6b14 158 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
a2e80f6f
PB
159 inflight[0] = mi.inflight[0];
160 inflight[1] = mi.inflight[1];
bf0ddaba
OS
161}
162
1671d522 163void blk_freeze_queue_start(struct request_queue *q)
43a5e4e2 164{
7996a8b5
BL
165 mutex_lock(&q->mq_freeze_lock);
166 if (++q->mq_freeze_depth == 1) {
3ef28e83 167 percpu_ref_kill(&q->q_usage_counter);
7996a8b5 168 mutex_unlock(&q->mq_freeze_lock);
344e9ffc 169 if (queue_is_mq(q))
055f6e18 170 blk_mq_run_hw_queues(q, false);
7996a8b5
BL
171 } else {
172 mutex_unlock(&q->mq_freeze_lock);
cddd5d17 173 }
f3af020b 174}
1671d522 175EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
f3af020b 176
6bae363e 177void blk_mq_freeze_queue_wait(struct request_queue *q)
f3af020b 178{
3ef28e83 179 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
43a5e4e2 180}
6bae363e 181EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
43a5e4e2 182
f91328c4
KB
183int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
184 unsigned long timeout)
185{
186 return wait_event_timeout(q->mq_freeze_wq,
187 percpu_ref_is_zero(&q->q_usage_counter),
188 timeout);
189}
190EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
43a5e4e2 191
f3af020b
TH
192/*
193 * Guarantee no request is in use, so we can change any data structure of
194 * the queue afterward.
195 */
3ef28e83 196void blk_freeze_queue(struct request_queue *q)
f3af020b 197{
3ef28e83
DW
198 /*
199 * In the !blk_mq case we are only calling this to kill the
200 * q_usage_counter, otherwise this increases the freeze depth
201 * and waits for it to return to zero. For this reason there is
202 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
203 * exported to drivers as the only user for unfreeze is blk_mq.
204 */
1671d522 205 blk_freeze_queue_start(q);
f3af020b
TH
206 blk_mq_freeze_queue_wait(q);
207}
3ef28e83
DW
208
209void blk_mq_freeze_queue(struct request_queue *q)
210{
211 /*
212 * ...just an alias to keep freeze and unfreeze actions balanced
213 * in the blk_mq_* namespace
214 */
215 blk_freeze_queue(q);
216}
c761d96b 217EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
f3af020b 218
aec89dc5 219void __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic)
320ae51f 220{
7996a8b5 221 mutex_lock(&q->mq_freeze_lock);
aec89dc5
CH
222 if (force_atomic)
223 q->q_usage_counter.data->force_atomic = true;
7996a8b5
BL
224 q->mq_freeze_depth--;
225 WARN_ON_ONCE(q->mq_freeze_depth < 0);
226 if (!q->mq_freeze_depth) {
bdd63160 227 percpu_ref_resurrect(&q->q_usage_counter);
320ae51f 228 wake_up_all(&q->mq_freeze_wq);
add703fd 229 }
7996a8b5 230 mutex_unlock(&q->mq_freeze_lock);
320ae51f 231}
aec89dc5
CH
232
233void blk_mq_unfreeze_queue(struct request_queue *q)
234{
235 __blk_mq_unfreeze_queue(q, false);
236}
b4c6a028 237EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
320ae51f 238
852ec809
BVA
239/*
240 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the
241 * mpt3sas driver such that this function can be removed.
242 */
243void blk_mq_quiesce_queue_nowait(struct request_queue *q)
244{
8814ce8a 245 blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q);
852ec809
BVA
246}
247EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);
248
6a83e74d 249/**
69e07c4a 250 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
6a83e74d
BVA
251 * @q: request queue.
252 *
253 * Note: this function does not prevent that the struct request end_io()
69e07c4a
ML
254 * callback function is invoked. Once this function is returned, we make
255 * sure no dispatch can happen until the queue is unquiesced via
256 * blk_mq_unquiesce_queue().
6a83e74d
BVA
257 */
258void blk_mq_quiesce_queue(struct request_queue *q)
259{
260 struct blk_mq_hw_ctx *hctx;
261 unsigned int i;
262 bool rcu = false;
263
1d9e9bc6 264 blk_mq_quiesce_queue_nowait(q);
f4560ffe 265
6a83e74d
BVA
266 queue_for_each_hw_ctx(q, hctx, i) {
267 if (hctx->flags & BLK_MQ_F_BLOCKING)
05707b64 268 synchronize_srcu(hctx->srcu);
6a83e74d
BVA
269 else
270 rcu = true;
271 }
272 if (rcu)
273 synchronize_rcu();
274}
275EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
276
e4e73913
ML
277/*
278 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
279 * @q: request queue.
280 *
281 * This function recovers queue into the state before quiescing
282 * which is done by blk_mq_quiesce_queue.
283 */
284void blk_mq_unquiesce_queue(struct request_queue *q)
285{
8814ce8a 286 blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
f4560ffe 287
1d9e9bc6
ML
288 /* dispatch requests which are inserted during quiescing */
289 blk_mq_run_hw_queues(q, true);
e4e73913
ML
290}
291EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);
292
aed3ea94
JA
293void blk_mq_wake_waiters(struct request_queue *q)
294{
295 struct blk_mq_hw_ctx *hctx;
296 unsigned int i;
297
298 queue_for_each_hw_ctx(q, hctx, i)
299 if (blk_mq_hw_queue_mapped(hctx))
300 blk_mq_tag_wakeup_all(hctx->tags, true);
301}
302
fe1f4526 303/*
9a91b05b
HT
304 * Only need start/end time stamping if we have iostat or
305 * blk stats enabled, or using an IO scheduler.
fe1f4526
JA
306 */
307static inline bool blk_mq_need_time_stamp(struct request *rq)
308{
9a91b05b 309 return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS)) || rq->q->elevator;
fe1f4526
JA
310}
311
e4cdf1a1 312static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
7ea4d8a4 313 unsigned int tag, u64 alloc_time_ns)
320ae51f 314{
e4cdf1a1
CH
315 struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
316 struct request *rq = tags->static_rqs[tag];
c3a148d2 317
42fdc5e4 318 if (data->q->elevator) {
76647368 319 rq->tag = BLK_MQ_NO_TAG;
e4cdf1a1
CH
320 rq->internal_tag = tag;
321 } else {
e4cdf1a1 322 rq->tag = tag;
76647368 323 rq->internal_tag = BLK_MQ_NO_TAG;
e4cdf1a1
CH
324 }
325
af76e555 326 /* csd/requeue_work/fifo_time is initialized before use */
e4cdf1a1
CH
327 rq->q = data->q;
328 rq->mq_ctx = data->ctx;
ea4f995e 329 rq->mq_hctx = data->hctx;
568f2700 330 rq->rq_flags = 0;
7ea4d8a4 331 rq->cmd_flags = data->cmd_flags;
0854bcdc
BVA
332 if (data->flags & BLK_MQ_REQ_PM)
333 rq->rq_flags |= RQF_PM;
e4cdf1a1 334 if (blk_queue_io_stat(data->q))
e8064021 335 rq->rq_flags |= RQF_IO_STAT;
7c3fb70f 336 INIT_LIST_HEAD(&rq->queuelist);
af76e555
CH
337 INIT_HLIST_NODE(&rq->hash);
338 RB_CLEAR_NODE(&rq->rb_node);
af76e555
CH
339 rq->rq_disk = NULL;
340 rq->part = NULL;
6f816b4b
TH
341#ifdef CONFIG_BLK_RQ_ALLOC_TIME
342 rq->alloc_time_ns = alloc_time_ns;
343#endif
fe1f4526
JA
344 if (blk_mq_need_time_stamp(rq))
345 rq->start_time_ns = ktime_get_ns();
346 else
347 rq->start_time_ns = 0;
544ccc8d 348 rq->io_start_time_ns = 0;
3d244306 349 rq->stats_sectors = 0;
af76e555
CH
350 rq->nr_phys_segments = 0;
351#if defined(CONFIG_BLK_DEV_INTEGRITY)
352 rq->nr_integrity_segments = 0;
353#endif
a892c8d5 354 blk_crypto_rq_set_defaults(rq);
af76e555 355 /* tag was already set */
079076b3 356 WRITE_ONCE(rq->deadline, 0);
af76e555 357
f6be4fb4
JA
358 rq->timeout = 0;
359
af76e555
CH
360 rq->end_io = NULL;
361 rq->end_io_data = NULL;
af76e555 362
7ea4d8a4 363 data->ctx->rq_dispatched[op_is_sync(data->cmd_flags)]++;
12f5b931 364 refcount_set(&rq->ref, 1);
7ea4d8a4
CH
365
366 if (!op_is_flush(data->cmd_flags)) {
367 struct elevator_queue *e = data->q->elevator;
368
369 rq->elv.icq = NULL;
370 if (e && e->type->ops.prepare_request) {
371 if (e->type->icq_cache)
372 blk_mq_sched_assign_ioc(rq);
373
374 e->type->ops.prepare_request(rq);
375 rq->rq_flags |= RQF_ELVPRIV;
376 }
377 }
378
379 data->hctx->queued++;
e4cdf1a1 380 return rq;
5dee8577
CH
381}
382
349302da
JA
383static inline struct request *
384__blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data,
385 u64 alloc_time_ns)
386{
387 unsigned int tag, tag_offset;
388 struct request *rq;
389 unsigned long tags;
390 int i, nr = 0;
391
392 tags = blk_mq_get_tags(data, data->nr_tags, &tag_offset);
393 if (unlikely(!tags))
394 return NULL;
395
396 for (i = 0; tags; i++) {
397 if (!(tags & (1UL << i)))
398 continue;
399 tag = tag_offset + i;
400 tags &= ~(1UL << i);
401 rq = blk_mq_rq_ctx_init(data, tag, alloc_time_ns);
402 rq->rq_next = *data->cached_rq;
403 *data->cached_rq = rq;
404 }
405 data->nr_tags -= nr;
406
407 if (!data->cached_rq)
408 return NULL;
409
410 rq = *data->cached_rq;
411 *data->cached_rq = rq->rq_next;
412 return rq;
413}
414
b90cfaed 415static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
d2c0d383 416{
e6e7abff 417 struct request_queue *q = data->q;
d2c0d383 418 struct elevator_queue *e = q->elevator;
6f816b4b 419 u64 alloc_time_ns = 0;
47c122e3 420 struct request *rq;
600c3b0c 421 unsigned int tag;
d2c0d383 422
6f816b4b
TH
423 /* alloc_time includes depth and tag waits */
424 if (blk_queue_rq_alloc_time(q))
425 alloc_time_ns = ktime_get_ns();
426
f9afca4d 427 if (data->cmd_flags & REQ_NOWAIT)
03a07c92 428 data->flags |= BLK_MQ_REQ_NOWAIT;
d2c0d383
CH
429
430 if (e) {
d2c0d383 431 /*
8d663f34 432 * Flush/passthrough requests are special and go directly to the
17a51199
JA
433 * dispatch list. Don't include reserved tags in the
434 * limiting, as it isn't useful.
d2c0d383 435 */
f9afca4d 436 if (!op_is_flush(data->cmd_flags) &&
8d663f34 437 !blk_op_is_passthrough(data->cmd_flags) &&
f9afca4d 438 e->type->ops.limit_depth &&
17a51199 439 !(data->flags & BLK_MQ_REQ_RESERVED))
f9afca4d 440 e->type->ops.limit_depth(data->cmd_flags, data);
d2c0d383
CH
441 }
442
bf0beec0 443retry:
600c3b0c
CH
444 data->ctx = blk_mq_get_ctx(q);
445 data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);
42fdc5e4 446 if (!e)
600c3b0c
CH
447 blk_mq_tag_busy(data->hctx);
448
349302da
JA
449 /*
450 * Try batched alloc if we want more than 1 tag.
451 */
452 if (data->nr_tags > 1) {
453 rq = __blk_mq_alloc_requests_batch(data, alloc_time_ns);
454 if (rq)
455 return rq;
456 data->nr_tags = 1;
457 }
458
bf0beec0
ML
459 /*
460 * Waiting allocations only fail because of an inactive hctx. In that
461 * case just retry the hctx assignment and tag allocation as CPU hotplug
462 * should have migrated us to an online CPU by now.
463 */
349302da
JA
464 tag = blk_mq_get_tag(data);
465 if (tag == BLK_MQ_NO_TAG) {
466 if (data->flags & BLK_MQ_REQ_NOWAIT)
467 return NULL;
468 /*
469 * Give up the CPU and sleep for a random short time to
470 * ensure that thread using a realtime scheduling class
471 * are migrated off the CPU, and thus off the hctx that
472 * is going away.
473 */
474 msleep(3);
475 goto retry;
476 }
47c122e3 477
349302da 478 return blk_mq_rq_ctx_init(data, tag, alloc_time_ns);
d2c0d383
CH
479}
480
cd6ce148 481struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op,
9a95e4ef 482 blk_mq_req_flags_t flags)
320ae51f 483{
e6e7abff
CH
484 struct blk_mq_alloc_data data = {
485 .q = q,
486 .flags = flags,
487 .cmd_flags = op,
47c122e3 488 .nr_tags = 1,
e6e7abff 489 };
bd166ef1 490 struct request *rq;
a492f075 491 int ret;
320ae51f 492
3a0a5299 493 ret = blk_queue_enter(q, flags);
a492f075
JL
494 if (ret)
495 return ERR_PTR(ret);
320ae51f 496
b90cfaed 497 rq = __blk_mq_alloc_requests(&data);
bd166ef1 498 if (!rq)
a5ea5811 499 goto out_queue_exit;
0c4de0f3
CH
500 rq->__data_len = 0;
501 rq->__sector = (sector_t) -1;
502 rq->bio = rq->biotail = NULL;
320ae51f 503 return rq;
a5ea5811
CH
504out_queue_exit:
505 blk_queue_exit(q);
506 return ERR_PTR(-EWOULDBLOCK);
320ae51f 507}
4bb659b1 508EXPORT_SYMBOL(blk_mq_alloc_request);
320ae51f 509
cd6ce148 510struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
9a95e4ef 511 unsigned int op, blk_mq_req_flags_t flags, unsigned int hctx_idx)
1f5bd336 512{
e6e7abff
CH
513 struct blk_mq_alloc_data data = {
514 .q = q,
515 .flags = flags,
516 .cmd_flags = op,
47c122e3 517 .nr_tags = 1,
e6e7abff 518 };
600c3b0c 519 u64 alloc_time_ns = 0;
6d2809d5 520 unsigned int cpu;
600c3b0c 521 unsigned int tag;
1f5bd336
ML
522 int ret;
523
600c3b0c
CH
524 /* alloc_time includes depth and tag waits */
525 if (blk_queue_rq_alloc_time(q))
526 alloc_time_ns = ktime_get_ns();
527
1f5bd336
ML
528 /*
529 * If the tag allocator sleeps we could get an allocation for a
530 * different hardware context. No need to complicate the low level
531 * allocator for this for the rare use case of a command tied to
532 * a specific queue.
533 */
600c3b0c 534 if (WARN_ON_ONCE(!(flags & (BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED))))
1f5bd336
ML
535 return ERR_PTR(-EINVAL);
536
537 if (hctx_idx >= q->nr_hw_queues)
538 return ERR_PTR(-EIO);
539
3a0a5299 540 ret = blk_queue_enter(q, flags);
1f5bd336
ML
541 if (ret)
542 return ERR_PTR(ret);
543
c8712c6a
CH
544 /*
545 * Check if the hardware context is actually mapped to anything.
546 * If not tell the caller that it should skip this queue.
547 */
a5ea5811 548 ret = -EXDEV;
e6e7abff
CH
549 data.hctx = q->queue_hw_ctx[hctx_idx];
550 if (!blk_mq_hw_queue_mapped(data.hctx))
a5ea5811 551 goto out_queue_exit;
e6e7abff
CH
552 cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
553 data.ctx = __blk_mq_get_ctx(q, cpu);
1f5bd336 554
42fdc5e4 555 if (!q->elevator)
600c3b0c
CH
556 blk_mq_tag_busy(data.hctx);
557
a5ea5811 558 ret = -EWOULDBLOCK;
600c3b0c
CH
559 tag = blk_mq_get_tag(&data);
560 if (tag == BLK_MQ_NO_TAG)
a5ea5811 561 goto out_queue_exit;
600c3b0c
CH
562 return blk_mq_rq_ctx_init(&data, tag, alloc_time_ns);
563
a5ea5811
CH
564out_queue_exit:
565 blk_queue_exit(q);
566 return ERR_PTR(ret);
1f5bd336
ML
567}
568EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
569
12f5b931
KB
570static void __blk_mq_free_request(struct request *rq)
571{
572 struct request_queue *q = rq->q;
573 struct blk_mq_ctx *ctx = rq->mq_ctx;
ea4f995e 574 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
12f5b931
KB
575 const int sched_tag = rq->internal_tag;
576
a892c8d5 577 blk_crypto_free_request(rq);
986d413b 578 blk_pm_mark_last_busy(rq);
ea4f995e 579 rq->mq_hctx = NULL;
76647368 580 if (rq->tag != BLK_MQ_NO_TAG)
cae740a0 581 blk_mq_put_tag(hctx->tags, ctx, rq->tag);
76647368 582 if (sched_tag != BLK_MQ_NO_TAG)
cae740a0 583 blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
12f5b931
KB
584 blk_mq_sched_restart(hctx);
585 blk_queue_exit(q);
586}
587
6af54051 588void blk_mq_free_request(struct request *rq)
320ae51f 589{
320ae51f 590 struct request_queue *q = rq->q;
6af54051
CH
591 struct elevator_queue *e = q->elevator;
592 struct blk_mq_ctx *ctx = rq->mq_ctx;
ea4f995e 593 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
6af54051 594
5bbf4e5a 595 if (rq->rq_flags & RQF_ELVPRIV) {
f9cd4bfe
JA
596 if (e && e->type->ops.finish_request)
597 e->type->ops.finish_request(rq);
6af54051
CH
598 if (rq->elv.icq) {
599 put_io_context(rq->elv.icq->ioc);
600 rq->elv.icq = NULL;
601 }
602 }
320ae51f 603
6af54051 604 ctx->rq_completed[rq_is_sync(rq)]++;
e8064021 605 if (rq->rq_flags & RQF_MQ_INFLIGHT)
bccf5e26 606 __blk_mq_dec_active_requests(hctx);
87760e5e 607
7beb2f84 608 if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
d152c682 609 laptop_io_completion(q->disk->bdi);
7beb2f84 610
a7905043 611 rq_qos_done(q, rq);
0d2602ca 612
12f5b931
KB
613 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
614 if (refcount_dec_and_test(&rq->ref))
615 __blk_mq_free_request(rq);
320ae51f 616}
1a3b595a 617EXPORT_SYMBOL_GPL(blk_mq_free_request);
320ae51f 618
47c122e3
JA
619void blk_mq_free_plug_rqs(struct blk_plug *plug)
620{
621 while (plug->cached_rq) {
622 struct request *rq;
623
624 rq = plug->cached_rq;
625 plug->cached_rq = rq->rq_next;
626 percpu_ref_get(&rq->q->q_usage_counter);
627 blk_mq_free_request(rq);
628 }
629}
630
2a842aca 631inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
320ae51f 632{
8971a3b7
PB
633 if (blk_mq_need_time_stamp(rq)) {
634 u64 now = ktime_get_ns();
fe1f4526 635
8971a3b7
PB
636 if (rq->rq_flags & RQF_STATS) {
637 blk_mq_poll_stats_start(rq->q);
638 blk_stat_add(rq, now);
639 }
522a7775 640
8971a3b7
PB
641 blk_mq_sched_completed_request(rq, now);
642 blk_account_io_done(rq, now);
4bc6339a
OS
643 }
644
91b63639 645 if (rq->end_io) {
a7905043 646 rq_qos_done(rq->q, rq);
320ae51f 647 rq->end_io(rq, error);
91b63639 648 } else {
320ae51f 649 blk_mq_free_request(rq);
91b63639 650 }
320ae51f 651}
c8a446ad 652EXPORT_SYMBOL(__blk_mq_end_request);
63151a44 653
2a842aca 654void blk_mq_end_request(struct request *rq, blk_status_t error)
63151a44
CH
655{
656 if (blk_update_request(rq, error, blk_rq_bytes(rq)))
657 BUG();
c8a446ad 658 __blk_mq_end_request(rq, error);
63151a44 659}
c8a446ad 660EXPORT_SYMBOL(blk_mq_end_request);
320ae51f 661
f9ab4918 662static void blk_complete_reqs(struct llist_head *list)
320ae51f 663{
f9ab4918
SAS
664 struct llist_node *entry = llist_reverse_order(llist_del_all(list));
665 struct request *rq, *next;
c3077b5d 666
f9ab4918 667 llist_for_each_entry_safe(rq, next, entry, ipi_list)
c3077b5d 668 rq->q->mq_ops->complete(rq);
320ae51f 669}
320ae51f 670
f9ab4918 671static __latent_entropy void blk_done_softirq(struct softirq_action *h)
320ae51f 672{
f9ab4918 673 blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));
115243f5
CH
674}
675
c3077b5d
CH
676static int blk_softirq_cpu_dead(unsigned int cpu)
677{
f9ab4918 678 blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
c3077b5d
CH
679 return 0;
680}
681
40d09b53 682static void __blk_mq_complete_request_remote(void *data)
c3077b5d 683{
f9ab4918 684 __raise_softirq_irqoff(BLOCK_SOFTIRQ);
c3077b5d
CH
685}
686
96339526
CH
687static inline bool blk_mq_complete_need_ipi(struct request *rq)
688{
689 int cpu = raw_smp_processor_id();
690
691 if (!IS_ENABLED(CONFIG_SMP) ||
692 !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
693 return false;
71425189
SAS
694 /*
695 * With force threaded interrupts enabled, raising softirq from an SMP
696 * function call will always result in waking the ksoftirqd thread.
697 * This is probably worse than completing the request on a different
698 * cache domain.
699 */
91cc470e 700 if (force_irqthreads())
71425189 701 return false;
96339526
CH
702
703 /* same CPU or cache domain? Complete locally */
704 if (cpu == rq->mq_ctx->cpu ||
705 (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
706 cpus_share_cache(cpu, rq->mq_ctx->cpu)))
707 return false;
708
709 /* don't try to IPI to an offline CPU */
710 return cpu_online(rq->mq_ctx->cpu);
711}
712
f9ab4918
SAS
713static void blk_mq_complete_send_ipi(struct request *rq)
714{
715 struct llist_head *list;
716 unsigned int cpu;
717
718 cpu = rq->mq_ctx->cpu;
719 list = &per_cpu(blk_cpu_done, cpu);
720 if (llist_add(&rq->ipi_list, list)) {
721 INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq);
722 smp_call_function_single_async(cpu, &rq->csd);
723 }
724}
725
726static void blk_mq_raise_softirq(struct request *rq)
727{
728 struct llist_head *list;
729
730 preempt_disable();
731 list = this_cpu_ptr(&blk_cpu_done);
732 if (llist_add(&rq->ipi_list, list))
733 raise_softirq(BLOCK_SOFTIRQ);
734 preempt_enable();
735}
736
40d09b53 737bool blk_mq_complete_request_remote(struct request *rq)
320ae51f 738{
af78ff7c 739 WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
36e76539 740
4ab32bf3
JA
741 /*
742 * For a polled request, always complete locallly, it's pointless
743 * to redirect the completion.
744 */
6ce913fe 745 if (rq->cmd_flags & REQ_POLLED)
40d09b53 746 return false;
38535201 747
96339526 748 if (blk_mq_complete_need_ipi(rq)) {
f9ab4918
SAS
749 blk_mq_complete_send_ipi(rq);
750 return true;
3d6efbf6 751 }
40d09b53 752
f9ab4918
SAS
753 if (rq->q->nr_hw_queues == 1) {
754 blk_mq_raise_softirq(rq);
755 return true;
756 }
757 return false;
40d09b53
CH
758}
759EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);
760
761/**
762 * blk_mq_complete_request - end I/O on a request
763 * @rq: the request being processed
764 *
765 * Description:
766 * Complete a request by scheduling the ->complete_rq operation.
767 **/
768void blk_mq_complete_request(struct request *rq)
769{
770 if (!blk_mq_complete_request_remote(rq))
771 rq->q->mq_ops->complete(rq);
320ae51f 772}
15f73f5b 773EXPORT_SYMBOL(blk_mq_complete_request);
30a91cb4 774
04ced159 775static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx)
b7435db8 776 __releases(hctx->srcu)
04ced159
JA
777{
778 if (!(hctx->flags & BLK_MQ_F_BLOCKING))
779 rcu_read_unlock();
780 else
05707b64 781 srcu_read_unlock(hctx->srcu, srcu_idx);
04ced159
JA
782}
783
784static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx)
b7435db8 785 __acquires(hctx->srcu)
04ced159 786{
08b5a6e2
JA
787 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
788 /* shut up gcc false positive */
789 *srcu_idx = 0;
04ced159 790 rcu_read_lock();
08b5a6e2 791 } else
05707b64 792 *srcu_idx = srcu_read_lock(hctx->srcu);
04ced159
JA
793}
794
105663f7
AA
795/**
796 * blk_mq_start_request - Start processing a request
797 * @rq: Pointer to request to be started
798 *
799 * Function used by device drivers to notify the block layer that a request
800 * is going to be processed now, so blk layer can do proper initializations
801 * such as starting the timeout timer.
802 */
e2490073 803void blk_mq_start_request(struct request *rq)
320ae51f
JA
804{
805 struct request_queue *q = rq->q;
806
a54895fa 807 trace_block_rq_issue(rq);
320ae51f 808
cf43e6be 809 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
00067077
JA
810 u64 start_time;
811#ifdef CONFIG_BLK_CGROUP
812 if (rq->bio)
813 start_time = bio_issue_time(&rq->bio->bi_issue);
814 else
815#endif
816 start_time = ktime_get_ns();
817 rq->io_start_time_ns = start_time;
3d244306 818 rq->stats_sectors = blk_rq_sectors(rq);
cf43e6be 819 rq->rq_flags |= RQF_STATS;
a7905043 820 rq_qos_issue(q, rq);
cf43e6be
JA
821 }
822
1d9bd516 823 WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
538b7534 824
1d9bd516 825 blk_add_timer(rq);
12f5b931 826 WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);
49f5baa5 827
54d4e6ab
MG
828#ifdef CONFIG_BLK_DEV_INTEGRITY
829 if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE)
830 q->integrity.profile->prepare_fn(rq);
831#endif
3e08773c
CH
832 if (rq->bio && rq->bio->bi_opf & REQ_POLLED)
833 WRITE_ONCE(rq->bio->bi_cookie, blk_rq_to_qc(rq));
320ae51f 834}
e2490073 835EXPORT_SYMBOL(blk_mq_start_request);
320ae51f 836
ed0791b2 837static void __blk_mq_requeue_request(struct request *rq)
320ae51f
JA
838{
839 struct request_queue *q = rq->q;
840
923218f6
ML
841 blk_mq_put_driver_tag(rq);
842
a54895fa 843 trace_block_rq_requeue(rq);
a7905043 844 rq_qos_requeue(q, rq);
49f5baa5 845
12f5b931
KB
846 if (blk_mq_request_started(rq)) {
847 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
da661267 848 rq->rq_flags &= ~RQF_TIMED_OUT;
e2490073 849 }
320ae51f
JA
850}
851
2b053aca 852void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
ed0791b2 853{
ed0791b2 854 __blk_mq_requeue_request(rq);
ed0791b2 855
105976f5
ML
856 /* this request will be re-inserted to io scheduler queue */
857 blk_mq_sched_requeue_request(rq);
858
7d692330 859 BUG_ON(!list_empty(&rq->queuelist));
2b053aca 860 blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
ed0791b2
CH
861}
862EXPORT_SYMBOL(blk_mq_requeue_request);
863
6fca6a61
CH
864static void blk_mq_requeue_work(struct work_struct *work)
865{
866 struct request_queue *q =
2849450a 867 container_of(work, struct request_queue, requeue_work.work);
6fca6a61
CH
868 LIST_HEAD(rq_list);
869 struct request *rq, *next;
6fca6a61 870
18e9781d 871 spin_lock_irq(&q->requeue_lock);
6fca6a61 872 list_splice_init(&q->requeue_list, &rq_list);
18e9781d 873 spin_unlock_irq(&q->requeue_lock);
6fca6a61
CH
874
875 list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
aef1897c 876 if (!(rq->rq_flags & (RQF_SOFTBARRIER | RQF_DONTPREP)))
6fca6a61
CH
877 continue;
878
e8064021 879 rq->rq_flags &= ~RQF_SOFTBARRIER;
6fca6a61 880 list_del_init(&rq->queuelist);
aef1897c
JW
881 /*
882 * If RQF_DONTPREP, rq has contained some driver specific
883 * data, so insert it to hctx dispatch list to avoid any
884 * merge.
885 */
886 if (rq->rq_flags & RQF_DONTPREP)
01e99aec 887 blk_mq_request_bypass_insert(rq, false, false);
aef1897c
JW
888 else
889 blk_mq_sched_insert_request(rq, true, false, false);
6fca6a61
CH
890 }
891
892 while (!list_empty(&rq_list)) {
893 rq = list_entry(rq_list.next, struct request, queuelist);
894 list_del_init(&rq->queuelist);
9e97d295 895 blk_mq_sched_insert_request(rq, false, false, false);
6fca6a61
CH
896 }
897
52d7f1b5 898 blk_mq_run_hw_queues(q, false);
6fca6a61
CH
899}
900
2b053aca
BVA
901void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
902 bool kick_requeue_list)
6fca6a61
CH
903{
904 struct request_queue *q = rq->q;
905 unsigned long flags;
906
907 /*
908 * We abuse this flag that is otherwise used by the I/O scheduler to
ff821d27 909 * request head insertion from the workqueue.
6fca6a61 910 */
e8064021 911 BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
6fca6a61
CH
912
913 spin_lock_irqsave(&q->requeue_lock, flags);
914 if (at_head) {
e8064021 915 rq->rq_flags |= RQF_SOFTBARRIER;
6fca6a61
CH
916 list_add(&rq->queuelist, &q->requeue_list);
917 } else {
918 list_add_tail(&rq->queuelist, &q->requeue_list);
919 }
920 spin_unlock_irqrestore(&q->requeue_lock, flags);
2b053aca
BVA
921
922 if (kick_requeue_list)
923 blk_mq_kick_requeue_list(q);
6fca6a61 924}
6fca6a61
CH
925
926void blk_mq_kick_requeue_list(struct request_queue *q)
927{
ae943d20 928 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
6fca6a61
CH
929}
930EXPORT_SYMBOL(blk_mq_kick_requeue_list);
931
2849450a
MS
932void blk_mq_delay_kick_requeue_list(struct request_queue *q,
933 unsigned long msecs)
934{
d4acf365
BVA
935 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
936 msecs_to_jiffies(msecs));
2849450a
MS
937}
938EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
939
0e62f51f
JA
940struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
941{
88c7b2b7
JA
942 if (tag < tags->nr_tags) {
943 prefetch(tags->rqs[tag]);
4ee86bab 944 return tags->rqs[tag];
88c7b2b7 945 }
4ee86bab
HR
946
947 return NULL;
24d2f903
CH
948}
949EXPORT_SYMBOL(blk_mq_tag_to_rq);
950
3c94d83c
JA
951static bool blk_mq_rq_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq,
952 void *priv, bool reserved)
ae879912
JA
953{
954 /*
05a4fed6 955 * If we find a request that isn't idle and the queue matches,
3c94d83c 956 * we know the queue is busy. Return false to stop the iteration.
ae879912 957 */
05a4fed6 958 if (blk_mq_request_started(rq) && rq->q == hctx->queue) {
ae879912
JA
959 bool *busy = priv;
960
961 *busy = true;
962 return false;
963 }
964
965 return true;
966}
967
3c94d83c 968bool blk_mq_queue_inflight(struct request_queue *q)
ae879912
JA
969{
970 bool busy = false;
971
3c94d83c 972 blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
ae879912
JA
973 return busy;
974}
3c94d83c 975EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
ae879912 976
358f70da 977static void blk_mq_rq_timed_out(struct request *req, bool reserved)
320ae51f 978{
da661267 979 req->rq_flags |= RQF_TIMED_OUT;
d1210d5a
CH
980 if (req->q->mq_ops->timeout) {
981 enum blk_eh_timer_return ret;
982
983 ret = req->q->mq_ops->timeout(req, reserved);
984 if (ret == BLK_EH_DONE)
985 return;
986 WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
46f92d42 987 }
d1210d5a
CH
988
989 blk_add_timer(req);
87ee7b11 990}
5b3f25fc 991
12f5b931 992static bool blk_mq_req_expired(struct request *rq, unsigned long *next)
81481eb4 993{
12f5b931 994 unsigned long deadline;
87ee7b11 995
12f5b931
KB
996 if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
997 return false;
da661267
CH
998 if (rq->rq_flags & RQF_TIMED_OUT)
999 return false;
a7af0af3 1000
079076b3 1001 deadline = READ_ONCE(rq->deadline);
12f5b931
KB
1002 if (time_after_eq(jiffies, deadline))
1003 return true;
a7af0af3 1004
12f5b931
KB
1005 if (*next == 0)
1006 *next = deadline;
1007 else if (time_after(*next, deadline))
1008 *next = deadline;
1009 return false;
87ee7b11
JA
1010}
1011
2e315dc0
ML
1012void blk_mq_put_rq_ref(struct request *rq)
1013{
a9ed27a7 1014 if (is_flush_rq(rq))
2e315dc0
ML
1015 rq->end_io(rq, 0);
1016 else if (refcount_dec_and_test(&rq->ref))
1017 __blk_mq_free_request(rq);
1018}
1019
7baa8572 1020static bool blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
1d9bd516
TH
1021 struct request *rq, void *priv, bool reserved)
1022{
12f5b931
KB
1023 unsigned long *next = priv;
1024
1025 /*
c797b40c
ML
1026 * blk_mq_queue_tag_busy_iter() has locked the request, so it cannot
1027 * be reallocated underneath the timeout handler's processing, then
1028 * the expire check is reliable. If the request is not expired, then
1029 * it was completed and reallocated as a new request after returning
1030 * from blk_mq_check_expired().
1d9bd516 1031 */
12f5b931 1032 if (blk_mq_req_expired(rq, next))
1d9bd516 1033 blk_mq_rq_timed_out(rq, reserved);
7baa8572 1034 return true;
1d9bd516
TH
1035}
1036
287922eb 1037static void blk_mq_timeout_work(struct work_struct *work)
320ae51f 1038{
287922eb
CH
1039 struct request_queue *q =
1040 container_of(work, struct request_queue, timeout_work);
12f5b931 1041 unsigned long next = 0;
1d9bd516 1042 struct blk_mq_hw_ctx *hctx;
81481eb4 1043 int i;
320ae51f 1044
71f79fb3
GKB
1045 /* A deadlock might occur if a request is stuck requiring a
1046 * timeout at the same time a queue freeze is waiting
1047 * completion, since the timeout code would not be able to
1048 * acquire the queue reference here.
1049 *
1050 * That's why we don't use blk_queue_enter here; instead, we use
1051 * percpu_ref_tryget directly, because we need to be able to
1052 * obtain a reference even in the short window between the queue
1053 * starting to freeze, by dropping the first reference in
1671d522 1054 * blk_freeze_queue_start, and the moment the last request is
71f79fb3
GKB
1055 * consumed, marked by the instant q_usage_counter reaches
1056 * zero.
1057 */
1058 if (!percpu_ref_tryget(&q->q_usage_counter))
287922eb
CH
1059 return;
1060
12f5b931 1061 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next);
320ae51f 1062
12f5b931
KB
1063 if (next != 0) {
1064 mod_timer(&q->timeout, next);
0d2602ca 1065 } else {
fcd36c36
BVA
1066 /*
1067 * Request timeouts are handled as a forward rolling timer. If
1068 * we end up here it means that no requests are pending and
1069 * also that no request has been pending for a while. Mark
1070 * each hctx as idle.
1071 */
f054b56c
ML
1072 queue_for_each_hw_ctx(q, hctx, i) {
1073 /* the hctx may be unmapped, so check it here */
1074 if (blk_mq_hw_queue_mapped(hctx))
1075 blk_mq_tag_idle(hctx);
1076 }
0d2602ca 1077 }
287922eb 1078 blk_queue_exit(q);
320ae51f
JA
1079}
1080
88459642
OS
1081struct flush_busy_ctx_data {
1082 struct blk_mq_hw_ctx *hctx;
1083 struct list_head *list;
1084};
1085
1086static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
1087{
1088 struct flush_busy_ctx_data *flush_data = data;
1089 struct blk_mq_hw_ctx *hctx = flush_data->hctx;
1090 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
c16d6b5a 1091 enum hctx_type type = hctx->type;
88459642 1092
88459642 1093 spin_lock(&ctx->lock);
c16d6b5a 1094 list_splice_tail_init(&ctx->rq_lists[type], flush_data->list);
e9a99a63 1095 sbitmap_clear_bit(sb, bitnr);
88459642
OS
1096 spin_unlock(&ctx->lock);
1097 return true;
1098}
1099
1429d7c9
JA
1100/*
1101 * Process software queues that have been marked busy, splicing them
1102 * to the for-dispatch
1103 */
2c3ad667 1104void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
1429d7c9 1105{
88459642
OS
1106 struct flush_busy_ctx_data data = {
1107 .hctx = hctx,
1108 .list = list,
1109 };
1429d7c9 1110
88459642 1111 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
1429d7c9 1112}
2c3ad667 1113EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
1429d7c9 1114
b347689f
ML
1115struct dispatch_rq_data {
1116 struct blk_mq_hw_ctx *hctx;
1117 struct request *rq;
1118};
1119
1120static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
1121 void *data)
1122{
1123 struct dispatch_rq_data *dispatch_data = data;
1124 struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;
1125 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
c16d6b5a 1126 enum hctx_type type = hctx->type;
b347689f
ML
1127
1128 spin_lock(&ctx->lock);
c16d6b5a
ML
1129 if (!list_empty(&ctx->rq_lists[type])) {
1130 dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next);
b347689f 1131 list_del_init(&dispatch_data->rq->queuelist);
c16d6b5a 1132 if (list_empty(&ctx->rq_lists[type]))
b347689f
ML
1133 sbitmap_clear_bit(sb, bitnr);
1134 }
1135 spin_unlock(&ctx->lock);
1136
1137 return !dispatch_data->rq;
1138}
1139
1140struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
1141 struct blk_mq_ctx *start)
1142{
f31967f0 1143 unsigned off = start ? start->index_hw[hctx->type] : 0;
b347689f
ML
1144 struct dispatch_rq_data data = {
1145 .hctx = hctx,
1146 .rq = NULL,
1147 };
1148
1149 __sbitmap_for_each_set(&hctx->ctx_map, off,
1150 dispatch_rq_from_ctx, &data);
1151
1152 return data.rq;
1153}
1154
703fd1c0
JA
1155static inline unsigned int queued_to_index(unsigned int queued)
1156{
1157 if (!queued)
1158 return 0;
1429d7c9 1159
703fd1c0 1160 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
1429d7c9
JA
1161}
1162
570e9b73
ML
1163static bool __blk_mq_get_driver_tag(struct request *rq)
1164{
ae0f1a73 1165 struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
570e9b73 1166 unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
570e9b73
ML
1167 int tag;
1168
568f2700
ML
1169 blk_mq_tag_busy(rq->mq_hctx);
1170
570e9b73 1171 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
ae0f1a73 1172 bt = &rq->mq_hctx->tags->breserved_tags;
570e9b73 1173 tag_offset = 0;
28500850
ML
1174 } else {
1175 if (!hctx_may_queue(rq->mq_hctx, bt))
1176 return false;
570e9b73
ML
1177 }
1178
570e9b73
ML
1179 tag = __sbitmap_queue_get(bt);
1180 if (tag == BLK_MQ_NO_TAG)
1181 return false;
1182
1183 rq->tag = tag + tag_offset;
570e9b73
ML
1184 return true;
1185}
1186
61347154 1187bool blk_mq_get_driver_tag(struct request *rq)
570e9b73 1188{
568f2700
ML
1189 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1190
1191 if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_get_driver_tag(rq))
1192 return false;
1193
51db1c37 1194 if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
568f2700
ML
1195 !(rq->rq_flags & RQF_MQ_INFLIGHT)) {
1196 rq->rq_flags |= RQF_MQ_INFLIGHT;
bccf5e26 1197 __blk_mq_inc_active_requests(hctx);
568f2700
ML
1198 }
1199 hctx->tags->rqs[rq->tag] = rq;
1200 return true;
570e9b73
ML
1201}
1202
eb619fdb
JA
1203static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
1204 int flags, void *key)
da55f2cc
OS
1205{
1206 struct blk_mq_hw_ctx *hctx;
1207
1208 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
1209
5815839b 1210 spin_lock(&hctx->dispatch_wait_lock);
e8618575
JA
1211 if (!list_empty(&wait->entry)) {
1212 struct sbitmap_queue *sbq;
1213
1214 list_del_init(&wait->entry);
ae0f1a73 1215 sbq = &hctx->tags->bitmap_tags;
e8618575
JA
1216 atomic_dec(&sbq->ws_active);
1217 }
5815839b
ML
1218 spin_unlock(&hctx->dispatch_wait_lock);
1219
da55f2cc
OS
1220 blk_mq_run_hw_queue(hctx, true);
1221 return 1;
1222}
1223
f906a6a0
JA
1224/*
1225 * Mark us waiting for a tag. For shared tags, this involves hooking us into
ee3e4de5
BVA
1226 * the tag wakeups. For non-shared tags, we can simply mark us needing a
1227 * restart. For both cases, take care to check the condition again after
f906a6a0
JA
1228 * marking us as waiting.
1229 */
2278d69f 1230static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
f906a6a0 1231 struct request *rq)
da55f2cc 1232{
ae0f1a73 1233 struct sbitmap_queue *sbq = &hctx->tags->bitmap_tags;
5815839b 1234 struct wait_queue_head *wq;
f906a6a0
JA
1235 wait_queue_entry_t *wait;
1236 bool ret;
da55f2cc 1237
51db1c37 1238 if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
684b7324 1239 blk_mq_sched_mark_restart_hctx(hctx);
f906a6a0 1240
c27d53fb
BVA
1241 /*
1242 * It's possible that a tag was freed in the window between the
1243 * allocation failure and adding the hardware queue to the wait
1244 * queue.
1245 *
1246 * Don't clear RESTART here, someone else could have set it.
1247 * At most this will cost an extra queue run.
1248 */
8ab6bb9e 1249 return blk_mq_get_driver_tag(rq);
eb619fdb 1250 }
eb619fdb 1251
2278d69f 1252 wait = &hctx->dispatch_wait;
c27d53fb
BVA
1253 if (!list_empty_careful(&wait->entry))
1254 return false;
1255
e8618575 1256 wq = &bt_wait_ptr(sbq, hctx)->wait;
5815839b
ML
1257
1258 spin_lock_irq(&wq->lock);
1259 spin_lock(&hctx->dispatch_wait_lock);
c27d53fb 1260 if (!list_empty(&wait->entry)) {
5815839b
ML
1261 spin_unlock(&hctx->dispatch_wait_lock);
1262 spin_unlock_irq(&wq->lock);
c27d53fb 1263 return false;
eb619fdb
JA
1264 }
1265
e8618575 1266 atomic_inc(&sbq->ws_active);
5815839b
ML
1267 wait->flags &= ~WQ_FLAG_EXCLUSIVE;
1268 __add_wait_queue(wq, wait);
c27d53fb 1269
da55f2cc 1270 /*
eb619fdb
JA
1271 * It's possible that a tag was freed in the window between the
1272 * allocation failure and adding the hardware queue to the wait
1273 * queue.
da55f2cc 1274 */
8ab6bb9e 1275 ret = blk_mq_get_driver_tag(rq);
c27d53fb 1276 if (!ret) {
5815839b
ML
1277 spin_unlock(&hctx->dispatch_wait_lock);
1278 spin_unlock_irq(&wq->lock);
c27d53fb 1279 return false;
eb619fdb 1280 }
c27d53fb
BVA
1281
1282 /*
1283 * We got a tag, remove ourselves from the wait queue to ensure
1284 * someone else gets the wakeup.
1285 */
c27d53fb 1286 list_del_init(&wait->entry);
e8618575 1287 atomic_dec(&sbq->ws_active);
5815839b
ML
1288 spin_unlock(&hctx->dispatch_wait_lock);
1289 spin_unlock_irq(&wq->lock);
c27d53fb
BVA
1290
1291 return true;
da55f2cc
OS
1292}
1293
6e768717
ML
1294#define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT 8
1295#define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR 4
1296/*
1297 * Update dispatch busy with the Exponential Weighted Moving Average(EWMA):
1298 * - EWMA is one simple way to compute running average value
1299 * - weight(7/8 and 1/8) is applied so that it can decrease exponentially
1300 * - take 4 as factor for avoiding to get too small(0) result, and this
1301 * factor doesn't matter because EWMA decreases exponentially
1302 */
1303static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)
1304{
1305 unsigned int ewma;
1306
6e768717
ML
1307 ewma = hctx->dispatch_busy;
1308
1309 if (!ewma && !busy)
1310 return;
1311
1312 ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1;
1313 if (busy)
1314 ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR;
1315 ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT;
1316
1317 hctx->dispatch_busy = ewma;
1318}
1319
86ff7c2a
ML
1320#define BLK_MQ_RESOURCE_DELAY 3 /* ms units */
1321
c92a4103
JT
1322static void blk_mq_handle_dev_resource(struct request *rq,
1323 struct list_head *list)
1324{
1325 struct request *next =
1326 list_first_entry_or_null(list, struct request, queuelist);
1327
1328 /*
1329 * If an I/O scheduler has been configured and we got a driver tag for
1330 * the next request already, free it.
1331 */
1332 if (next)
1333 blk_mq_put_driver_tag(next);
1334
1335 list_add(&rq->queuelist, list);
1336 __blk_mq_requeue_request(rq);
1337}
1338
0512a75b
KB
1339static void blk_mq_handle_zone_resource(struct request *rq,
1340 struct list_head *zone_list)
1341{
1342 /*
1343 * If we end up here it is because we cannot dispatch a request to a
1344 * specific zone due to LLD level zone-write locking or other zone
1345 * related resource not being available. In this case, set the request
1346 * aside in zone_list for retrying it later.
1347 */
1348 list_add(&rq->queuelist, zone_list);
1349 __blk_mq_requeue_request(rq);
1350}
1351
75383524
ML
1352enum prep_dispatch {
1353 PREP_DISPATCH_OK,
1354 PREP_DISPATCH_NO_TAG,
1355 PREP_DISPATCH_NO_BUDGET,
1356};
1357
1358static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
1359 bool need_budget)
1360{
1361 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2a5a24aa 1362 int budget_token = -1;
75383524 1363
2a5a24aa
ML
1364 if (need_budget) {
1365 budget_token = blk_mq_get_dispatch_budget(rq->q);
1366 if (budget_token < 0) {
1367 blk_mq_put_driver_tag(rq);
1368 return PREP_DISPATCH_NO_BUDGET;
1369 }
1370 blk_mq_set_rq_budget_token(rq, budget_token);
75383524
ML
1371 }
1372
1373 if (!blk_mq_get_driver_tag(rq)) {
1374 /*
1375 * The initial allocation attempt failed, so we need to
1376 * rerun the hardware queue when a tag is freed. The
1377 * waitqueue takes care of that. If the queue is run
1378 * before we add this entry back on the dispatch list,
1379 * we'll re-run it below.
1380 */
1381 if (!blk_mq_mark_tag_wait(hctx, rq)) {
1fd40b5e
ML
1382 /*
1383 * All budgets not got from this function will be put
1384 * together during handling partial dispatch
1385 */
1386 if (need_budget)
2a5a24aa 1387 blk_mq_put_dispatch_budget(rq->q, budget_token);
75383524
ML
1388 return PREP_DISPATCH_NO_TAG;
1389 }
1390 }
1391
1392 return PREP_DISPATCH_OK;
1393}
1394
1fd40b5e
ML
1395/* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
1396static void blk_mq_release_budgets(struct request_queue *q,
2a5a24aa 1397 struct list_head *list)
1fd40b5e 1398{
2a5a24aa 1399 struct request *rq;
1fd40b5e 1400
2a5a24aa
ML
1401 list_for_each_entry(rq, list, queuelist) {
1402 int budget_token = blk_mq_get_rq_budget_token(rq);
1fd40b5e 1403
2a5a24aa
ML
1404 if (budget_token >= 0)
1405 blk_mq_put_dispatch_budget(q, budget_token);
1406 }
1fd40b5e
ML
1407}
1408
1f57f8d4
JA
1409/*
1410 * Returns true if we did some work AND can potentially do more.
1411 */
445874e8 1412bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
1fd40b5e 1413 unsigned int nr_budgets)
320ae51f 1414{
75383524 1415 enum prep_dispatch prep;
445874e8 1416 struct request_queue *q = hctx->queue;
6d6f167c 1417 struct request *rq, *nxt;
fc17b653 1418 int errors, queued;
86ff7c2a 1419 blk_status_t ret = BLK_STS_OK;
0512a75b 1420 LIST_HEAD(zone_list);
320ae51f 1421
81380ca1
OS
1422 if (list_empty(list))
1423 return false;
1424
320ae51f
JA
1425 /*
1426 * Now process all the entries, sending them to the driver.
1427 */
93efe981 1428 errors = queued = 0;
81380ca1 1429 do {
74c45052 1430 struct blk_mq_queue_data bd;
320ae51f 1431
f04c3df3 1432 rq = list_first_entry(list, struct request, queuelist);
0bca799b 1433
445874e8 1434 WARN_ON_ONCE(hctx != rq->mq_hctx);
1fd40b5e 1435 prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets);
75383524 1436 if (prep != PREP_DISPATCH_OK)
0bca799b 1437 break;
de148297 1438
320ae51f 1439 list_del_init(&rq->queuelist);
320ae51f 1440
74c45052 1441 bd.rq = rq;
113285b4
JA
1442
1443 /*
1444 * Flag last if we have no more requests, or if we have more
1445 * but can't assign a driver tag to it.
1446 */
1447 if (list_empty(list))
1448 bd.last = true;
1449 else {
113285b4 1450 nxt = list_first_entry(list, struct request, queuelist);
8ab6bb9e 1451 bd.last = !blk_mq_get_driver_tag(nxt);
113285b4 1452 }
74c45052 1453
1fd40b5e
ML
1454 /*
1455 * once the request is queued to lld, no need to cover the
1456 * budget any more
1457 */
1458 if (nr_budgets)
1459 nr_budgets--;
74c45052 1460 ret = q->mq_ops->queue_rq(hctx, &bd);
7bf13729
ML
1461 switch (ret) {
1462 case BLK_STS_OK:
1463 queued++;
320ae51f 1464 break;
7bf13729
ML
1465 case BLK_STS_RESOURCE:
1466 case BLK_STS_DEV_RESOURCE:
1467 blk_mq_handle_dev_resource(rq, list);
1468 goto out;
1469 case BLK_STS_ZONE_RESOURCE:
0512a75b
KB
1470 /*
1471 * Move the request to zone_list and keep going through
1472 * the dispatch list to find more requests the drive can
1473 * accept.
1474 */
1475 blk_mq_handle_zone_resource(rq, &zone_list);
7bf13729
ML
1476 break;
1477 default:
93efe981 1478 errors++;
e21ee5a6 1479 blk_mq_end_request(rq, ret);
320ae51f 1480 }
81380ca1 1481 } while (!list_empty(list));
7bf13729 1482out:
0512a75b
KB
1483 if (!list_empty(&zone_list))
1484 list_splice_tail_init(&zone_list, list);
1485
703fd1c0 1486 hctx->dispatched[queued_to_index(queued)]++;
320ae51f 1487
632bfb63 1488 /* If we didn't flush the entire list, we could have told the driver
1489 * there was more coming, but that turned out to be a lie.
1490 */
1491 if ((!list_empty(list) || errors) && q->mq_ops->commit_rqs && queued)
1492 q->mq_ops->commit_rqs(hctx);
320ae51f
JA
1493 /*
1494 * Any items that need requeuing? Stuff them into hctx->dispatch,
1495 * that is where we will continue on next queue run.
1496 */
f04c3df3 1497 if (!list_empty(list)) {
86ff7c2a 1498 bool needs_restart;
75383524
ML
1499 /* For non-shared tags, the RESTART check will suffice */
1500 bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
51db1c37 1501 (hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED);
75383524 1502 bool no_budget_avail = prep == PREP_DISPATCH_NO_BUDGET;
86ff7c2a 1503
2a5a24aa
ML
1504 if (nr_budgets)
1505 blk_mq_release_budgets(q, list);
86ff7c2a 1506
320ae51f 1507 spin_lock(&hctx->lock);
01e99aec 1508 list_splice_tail_init(list, &hctx->dispatch);
320ae51f 1509 spin_unlock(&hctx->lock);
f04c3df3 1510
d7d8535f
ML
1511 /*
1512 * Order adding requests to hctx->dispatch and checking
1513 * SCHED_RESTART flag. The pair of this smp_mb() is the one
1514 * in blk_mq_sched_restart(). Avoid restart code path to
1515 * miss the new added requests to hctx->dispatch, meantime
1516 * SCHED_RESTART is observed here.
1517 */
1518 smp_mb();
1519
9ba52e58 1520 /*
710c785f
BVA
1521 * If SCHED_RESTART was set by the caller of this function and
1522 * it is no longer set that means that it was cleared by another
1523 * thread and hence that a queue rerun is needed.
9ba52e58 1524 *
eb619fdb
JA
1525 * If 'no_tag' is set, that means that we failed getting
1526 * a driver tag with an I/O scheduler attached. If our dispatch
1527 * waitqueue is no longer active, ensure that we run the queue
1528 * AFTER adding our entries back to the list.
bd166ef1 1529 *
710c785f
BVA
1530 * If no I/O scheduler has been configured it is possible that
1531 * the hardware queue got stopped and restarted before requests
1532 * were pushed back onto the dispatch list. Rerun the queue to
1533 * avoid starvation. Notes:
1534 * - blk_mq_run_hw_queue() checks whether or not a queue has
1535 * been stopped before rerunning a queue.
1536 * - Some but not all block drivers stop a queue before
fc17b653 1537 * returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
710c785f 1538 * and dm-rq.
86ff7c2a
ML
1539 *
1540 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART
1541 * bit is set, run queue after a delay to avoid IO stalls
ab3cee37
DA
1542 * that could otherwise occur if the queue is idle. We'll do
1543 * similar if we couldn't get budget and SCHED_RESTART is set.
bd166ef1 1544 */
86ff7c2a
ML
1545 needs_restart = blk_mq_sched_needs_restart(hctx);
1546 if (!needs_restart ||
eb619fdb 1547 (no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
bd166ef1 1548 blk_mq_run_hw_queue(hctx, true);
ab3cee37
DA
1549 else if (needs_restart && (ret == BLK_STS_RESOURCE ||
1550 no_budget_avail))
86ff7c2a 1551 blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY);
1f57f8d4 1552
6e768717 1553 blk_mq_update_dispatch_busy(hctx, true);
1f57f8d4 1554 return false;
6e768717
ML
1555 } else
1556 blk_mq_update_dispatch_busy(hctx, false);
f04c3df3 1557
93efe981 1558 return (queued + errors) != 0;
f04c3df3
JA
1559}
1560
105663f7
AA
1561/**
1562 * __blk_mq_run_hw_queue - Run a hardware queue.
1563 * @hctx: Pointer to the hardware queue to run.
1564 *
1565 * Send pending requests to the hardware.
1566 */
6a83e74d
BVA
1567static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
1568{
1569 int srcu_idx;
1570
b7a71e66
JA
1571 /*
1572 * We can't run the queue inline with ints disabled. Ensure that
1573 * we catch bad users of this early.
1574 */
1575 WARN_ON_ONCE(in_interrupt());
1576
04ced159 1577 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
bf4907c0 1578
04ced159
JA
1579 hctx_lock(hctx, &srcu_idx);
1580 blk_mq_sched_dispatch_requests(hctx);
1581 hctx_unlock(hctx, srcu_idx);
6a83e74d
BVA
1582}
1583
f82ddf19
ML
1584static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
1585{
1586 int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);
1587
1588 if (cpu >= nr_cpu_ids)
1589 cpu = cpumask_first(hctx->cpumask);
1590 return cpu;
1591}
1592
506e931f
JA
1593/*
1594 * It'd be great if the workqueue API had a way to pass
1595 * in a mask and had some smarts for more clever placement.
1596 * For now we just round-robin here, switching for every
1597 * BLK_MQ_CPU_WORK_BATCH queued items.
1598 */
1599static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
1600{
7bed4595 1601 bool tried = false;
476f8c98 1602 int next_cpu = hctx->next_cpu;
7bed4595 1603
b657d7e6
CH
1604 if (hctx->queue->nr_hw_queues == 1)
1605 return WORK_CPU_UNBOUND;
506e931f
JA
1606
1607 if (--hctx->next_cpu_batch <= 0) {
7bed4595 1608select_cpu:
476f8c98 1609 next_cpu = cpumask_next_and(next_cpu, hctx->cpumask,
20e4d813 1610 cpu_online_mask);
506e931f 1611 if (next_cpu >= nr_cpu_ids)
f82ddf19 1612 next_cpu = blk_mq_first_mapped_cpu(hctx);
506e931f
JA
1613 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1614 }
1615
7bed4595
ML
1616 /*
1617 * Do unbound schedule if we can't find a online CPU for this hctx,
1618 * and it should only happen in the path of handling CPU DEAD.
1619 */
476f8c98 1620 if (!cpu_online(next_cpu)) {
7bed4595
ML
1621 if (!tried) {
1622 tried = true;
1623 goto select_cpu;
1624 }
1625
1626 /*
1627 * Make sure to re-select CPU next time once after CPUs
1628 * in hctx->cpumask become online again.
1629 */
476f8c98 1630 hctx->next_cpu = next_cpu;
7bed4595
ML
1631 hctx->next_cpu_batch = 1;
1632 return WORK_CPU_UNBOUND;
1633 }
476f8c98
ML
1634
1635 hctx->next_cpu = next_cpu;
1636 return next_cpu;
506e931f
JA
1637}
1638
105663f7
AA
1639/**
1640 * __blk_mq_delay_run_hw_queue - Run (or schedule to run) a hardware queue.
1641 * @hctx: Pointer to the hardware queue to run.
1642 * @async: If we want to run the queue asynchronously.
fa94ba8a 1643 * @msecs: Milliseconds of delay to wait before running the queue.
105663f7
AA
1644 *
1645 * If !@async, try to run the queue now. Else, run the queue asynchronously and
1646 * with a delay of @msecs.
1647 */
7587a5ae
BVA
1648static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async,
1649 unsigned long msecs)
320ae51f 1650{
5435c023 1651 if (unlikely(blk_mq_hctx_stopped(hctx)))
320ae51f
JA
1652 return;
1653
1b792f2f 1654 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
2a90d4aa
PB
1655 int cpu = get_cpu();
1656 if (cpumask_test_cpu(cpu, hctx->cpumask)) {
398205b8 1657 __blk_mq_run_hw_queue(hctx);
2a90d4aa 1658 put_cpu();
398205b8
PB
1659 return;
1660 }
e4043dcf 1661
2a90d4aa 1662 put_cpu();
e4043dcf 1663 }
398205b8 1664
ae943d20
BVA
1665 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
1666 msecs_to_jiffies(msecs));
7587a5ae
BVA
1667}
1668
105663f7
AA
1669/**
1670 * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously.
1671 * @hctx: Pointer to the hardware queue to run.
fa94ba8a 1672 * @msecs: Milliseconds of delay to wait before running the queue.
105663f7
AA
1673 *
1674 * Run a hardware queue asynchronously with a delay of @msecs.
1675 */
7587a5ae
BVA
1676void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
1677{
1678 __blk_mq_delay_run_hw_queue(hctx, true, msecs);
1679}
1680EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
1681
105663f7
AA
1682/**
1683 * blk_mq_run_hw_queue - Start to run a hardware queue.
1684 * @hctx: Pointer to the hardware queue to run.
1685 * @async: If we want to run the queue asynchronously.
1686 *
1687 * Check if the request queue is not in a quiesced state and if there are
1688 * pending requests to be sent. If this is true, run the queue to send requests
1689 * to hardware.
1690 */
626fb735 1691void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
7587a5ae 1692{
24f5a90f
ML
1693 int srcu_idx;
1694 bool need_run;
1695
1696 /*
1697 * When queue is quiesced, we may be switching io scheduler, or
1698 * updating nr_hw_queues, or other things, and we can't run queue
1699 * any more, even __blk_mq_hctx_has_pending() can't be called safely.
1700 *
1701 * And queue will be rerun in blk_mq_unquiesce_queue() if it is
1702 * quiesced.
1703 */
04ced159
JA
1704 hctx_lock(hctx, &srcu_idx);
1705 need_run = !blk_queue_quiesced(hctx->queue) &&
1706 blk_mq_hctx_has_pending(hctx);
1707 hctx_unlock(hctx, srcu_idx);
24f5a90f 1708
626fb735 1709 if (need_run)
79f720a7 1710 __blk_mq_delay_run_hw_queue(hctx, async, 0);
320ae51f 1711}
5b727272 1712EXPORT_SYMBOL(blk_mq_run_hw_queue);
320ae51f 1713
b6e68ee8
JK
1714/*
1715 * Is the request queue handled by an IO scheduler that does not respect
1716 * hardware queues when dispatching?
1717 */
1718static bool blk_mq_has_sqsched(struct request_queue *q)
1719{
1720 struct elevator_queue *e = q->elevator;
1721
1722 if (e && e->type->ops.dispatch_request &&
1723 !(e->type->elevator_features & ELEVATOR_F_MQ_AWARE))
1724 return true;
1725 return false;
1726}
1727
1728/*
1729 * Return prefered queue to dispatch from (if any) for non-mq aware IO
1730 * scheduler.
1731 */
1732static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)
1733{
1734 struct blk_mq_hw_ctx *hctx;
1735
1736 /*
1737 * If the IO scheduler does not respect hardware queues when
1738 * dispatching, we just don't bother with multiple HW queues and
1739 * dispatch from hctx for the current CPU since running multiple queues
1740 * just causes lock contention inside the scheduler and pointless cache
1741 * bouncing.
1742 */
1743 hctx = blk_mq_map_queue_type(q, HCTX_TYPE_DEFAULT,
1744 raw_smp_processor_id());
1745 if (!blk_mq_hctx_stopped(hctx))
1746 return hctx;
1747 return NULL;
1748}
1749
105663f7 1750/**
24f7bb88 1751 * blk_mq_run_hw_queues - Run all hardware queues in a request queue.
105663f7
AA
1752 * @q: Pointer to the request queue to run.
1753 * @async: If we want to run the queue asynchronously.
1754 */
b94ec296 1755void blk_mq_run_hw_queues(struct request_queue *q, bool async)
320ae51f 1756{
b6e68ee8 1757 struct blk_mq_hw_ctx *hctx, *sq_hctx;
320ae51f
JA
1758 int i;
1759
b6e68ee8
JK
1760 sq_hctx = NULL;
1761 if (blk_mq_has_sqsched(q))
1762 sq_hctx = blk_mq_get_sq_hctx(q);
320ae51f 1763 queue_for_each_hw_ctx(q, hctx, i) {
79f720a7 1764 if (blk_mq_hctx_stopped(hctx))
320ae51f 1765 continue;
b6e68ee8
JK
1766 /*
1767 * Dispatch from this hctx either if there's no hctx preferred
1768 * by IO scheduler or if it has requests that bypass the
1769 * scheduler.
1770 */
1771 if (!sq_hctx || sq_hctx == hctx ||
1772 !list_empty_careful(&hctx->dispatch))
1773 blk_mq_run_hw_queue(hctx, async);
320ae51f
JA
1774 }
1775}
b94ec296 1776EXPORT_SYMBOL(blk_mq_run_hw_queues);
320ae51f 1777
b9151e7b
DA
1778/**
1779 * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously.
1780 * @q: Pointer to the request queue to run.
fa94ba8a 1781 * @msecs: Milliseconds of delay to wait before running the queues.
b9151e7b
DA
1782 */
1783void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
1784{
b6e68ee8 1785 struct blk_mq_hw_ctx *hctx, *sq_hctx;
b9151e7b
DA
1786 int i;
1787
b6e68ee8
JK
1788 sq_hctx = NULL;
1789 if (blk_mq_has_sqsched(q))
1790 sq_hctx = blk_mq_get_sq_hctx(q);
b9151e7b
DA
1791 queue_for_each_hw_ctx(q, hctx, i) {
1792 if (blk_mq_hctx_stopped(hctx))
1793 continue;
b6e68ee8
JK
1794 /*
1795 * Dispatch from this hctx either if there's no hctx preferred
1796 * by IO scheduler or if it has requests that bypass the
1797 * scheduler.
1798 */
1799 if (!sq_hctx || sq_hctx == hctx ||
1800 !list_empty_careful(&hctx->dispatch))
1801 blk_mq_delay_run_hw_queue(hctx, msecs);
b9151e7b
DA
1802 }
1803}
1804EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);
1805
fd001443
BVA
1806/**
1807 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
1808 * @q: request queue.
1809 *
1810 * The caller is responsible for serializing this function against
1811 * blk_mq_{start,stop}_hw_queue().
1812 */
1813bool blk_mq_queue_stopped(struct request_queue *q)
1814{
1815 struct blk_mq_hw_ctx *hctx;
1816 int i;
1817
1818 queue_for_each_hw_ctx(q, hctx, i)
1819 if (blk_mq_hctx_stopped(hctx))
1820 return true;
1821
1822 return false;
1823}
1824EXPORT_SYMBOL(blk_mq_queue_stopped);
1825
39a70c76
ML
1826/*
1827 * This function is often used for pausing .queue_rq() by driver when
1828 * there isn't enough resource or some conditions aren't satisfied, and
4d606219 1829 * BLK_STS_RESOURCE is usually returned.
39a70c76
ML
1830 *
1831 * We do not guarantee that dispatch can be drained or blocked
1832 * after blk_mq_stop_hw_queue() returns. Please use
1833 * blk_mq_quiesce_queue() for that requirement.
1834 */
2719aa21
JA
1835void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
1836{
641a9ed6 1837 cancel_delayed_work(&hctx->run_work);
280d45f6 1838
641a9ed6 1839 set_bit(BLK_MQ_S_STOPPED, &hctx->state);
2719aa21 1840}
641a9ed6 1841EXPORT_SYMBOL(blk_mq_stop_hw_queue);
2719aa21 1842
39a70c76
ML
1843/*
1844 * This function is often used for pausing .queue_rq() by driver when
1845 * there isn't enough resource or some conditions aren't satisfied, and
4d606219 1846 * BLK_STS_RESOURCE is usually returned.
39a70c76
ML
1847 *
1848 * We do not guarantee that dispatch can be drained or blocked
1849 * after blk_mq_stop_hw_queues() returns. Please use
1850 * blk_mq_quiesce_queue() for that requirement.
1851 */
2719aa21
JA
1852void blk_mq_stop_hw_queues(struct request_queue *q)
1853{
641a9ed6
ML
1854 struct blk_mq_hw_ctx *hctx;
1855 int i;
1856
1857 queue_for_each_hw_ctx(q, hctx, i)
1858 blk_mq_stop_hw_queue(hctx);
280d45f6
CH
1859}
1860EXPORT_SYMBOL(blk_mq_stop_hw_queues);
1861
320ae51f
JA
1862void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
1863{
1864 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
e4043dcf 1865
0ffbce80 1866 blk_mq_run_hw_queue(hctx, false);
320ae51f
JA
1867}
1868EXPORT_SYMBOL(blk_mq_start_hw_queue);
1869
2f268556
CH
1870void blk_mq_start_hw_queues(struct request_queue *q)
1871{
1872 struct blk_mq_hw_ctx *hctx;
1873 int i;
1874
1875 queue_for_each_hw_ctx(q, hctx, i)
1876 blk_mq_start_hw_queue(hctx);
1877}
1878EXPORT_SYMBOL(blk_mq_start_hw_queues);
1879
ae911c5e
JA
1880void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1881{
1882 if (!blk_mq_hctx_stopped(hctx))
1883 return;
1884
1885 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1886 blk_mq_run_hw_queue(hctx, async);
1887}
1888EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
1889
1b4a3258 1890void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
320ae51f
JA
1891{
1892 struct blk_mq_hw_ctx *hctx;
1893 int i;
1894
ae911c5e
JA
1895 queue_for_each_hw_ctx(q, hctx, i)
1896 blk_mq_start_stopped_hw_queue(hctx, async);
320ae51f
JA
1897}
1898EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
1899
70f4db63 1900static void blk_mq_run_work_fn(struct work_struct *work)
320ae51f
JA
1901{
1902 struct blk_mq_hw_ctx *hctx;
1903
9f993737 1904 hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
320ae51f 1905
21c6e939 1906 /*
15fe8a90 1907 * If we are stopped, don't run the queue.
21c6e939 1908 */
0841031a 1909 if (blk_mq_hctx_stopped(hctx))
0196d6b4 1910 return;
7587a5ae
BVA
1911
1912 __blk_mq_run_hw_queue(hctx);
1913}
1914
cfd0c552 1915static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
cfd0c552
ML
1916 struct request *rq,
1917 bool at_head)
320ae51f 1918{
e57690fe 1919 struct blk_mq_ctx *ctx = rq->mq_ctx;
c16d6b5a 1920 enum hctx_type type = hctx->type;
e57690fe 1921
7b607814
BVA
1922 lockdep_assert_held(&ctx->lock);
1923
a54895fa 1924 trace_block_rq_insert(rq);
01b983c9 1925
72a0a36e 1926 if (at_head)
c16d6b5a 1927 list_add(&rq->queuelist, &ctx->rq_lists[type]);
72a0a36e 1928 else
c16d6b5a 1929 list_add_tail(&rq->queuelist, &ctx->rq_lists[type]);
cfd0c552 1930}
4bb659b1 1931
2c3ad667
JA
1932void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
1933 bool at_head)
cfd0c552
ML
1934{
1935 struct blk_mq_ctx *ctx = rq->mq_ctx;
1936
7b607814
BVA
1937 lockdep_assert_held(&ctx->lock);
1938
e57690fe 1939 __blk_mq_insert_req_list(hctx, rq, at_head);
320ae51f 1940 blk_mq_hctx_mark_pending(hctx, ctx);
320ae51f
JA
1941}
1942
105663f7
AA
1943/**
1944 * blk_mq_request_bypass_insert - Insert a request at dispatch list.
1945 * @rq: Pointer to request to be inserted.
26bfeb26 1946 * @at_head: true if the request should be inserted at the head of the list.
105663f7
AA
1947 * @run_queue: If we should run the hardware queue after inserting the request.
1948 *
157f377b
JA
1949 * Should only be used carefully, when the caller knows we want to
1950 * bypass a potential IO scheduler on the target device.
1951 */
01e99aec
ML
1952void blk_mq_request_bypass_insert(struct request *rq, bool at_head,
1953 bool run_queue)
157f377b 1954{
ea4f995e 1955 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
157f377b
JA
1956
1957 spin_lock(&hctx->lock);
01e99aec
ML
1958 if (at_head)
1959 list_add(&rq->queuelist, &hctx->dispatch);
1960 else
1961 list_add_tail(&rq->queuelist, &hctx->dispatch);
157f377b
JA
1962 spin_unlock(&hctx->lock);
1963
b0850297
ML
1964 if (run_queue)
1965 blk_mq_run_hw_queue(hctx, false);
157f377b
JA
1966}
1967
bd166ef1
JA
1968void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
1969 struct list_head *list)
320ae51f
JA
1970
1971{
3f0cedc7 1972 struct request *rq;
c16d6b5a 1973 enum hctx_type type = hctx->type;
3f0cedc7 1974
320ae51f
JA
1975 /*
1976 * preemption doesn't flush plug list, so it's possible ctx->cpu is
1977 * offline now
1978 */
3f0cedc7 1979 list_for_each_entry(rq, list, queuelist) {
e57690fe 1980 BUG_ON(rq->mq_ctx != ctx);
a54895fa 1981 trace_block_rq_insert(rq);
320ae51f 1982 }
3f0cedc7
ML
1983
1984 spin_lock(&ctx->lock);
c16d6b5a 1985 list_splice_tail_init(list, &ctx->rq_lists[type]);
cfd0c552 1986 blk_mq_hctx_mark_pending(hctx, ctx);
320ae51f 1987 spin_unlock(&ctx->lock);
320ae51f
JA
1988}
1989
4f0f586b
ST
1990static int plug_rq_cmp(void *priv, const struct list_head *a,
1991 const struct list_head *b)
320ae51f
JA
1992{
1993 struct request *rqa = container_of(a, struct request, queuelist);
1994 struct request *rqb = container_of(b, struct request, queuelist);
1995
7d30a621
PB
1996 if (rqa->mq_ctx != rqb->mq_ctx)
1997 return rqa->mq_ctx > rqb->mq_ctx;
1998 if (rqa->mq_hctx != rqb->mq_hctx)
1999 return rqa->mq_hctx > rqb->mq_hctx;
3110fc79
JA
2000
2001 return blk_rq_pos(rqa) > blk_rq_pos(rqb);
320ae51f
JA
2002}
2003
2004void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
2005{
320ae51f 2006 LIST_HEAD(list);
320ae51f 2007
95ed0c5b
PB
2008 if (list_empty(&plug->mq_list))
2009 return;
320ae51f
JA
2010 list_splice_init(&plug->mq_list, &list);
2011
ce5b009c
JA
2012 if (plug->rq_count > 2 && plug->multiple_queues)
2013 list_sort(NULL, &list, plug_rq_cmp);
320ae51f 2014
bcc816df
DZ
2015 plug->rq_count = 0;
2016
95ed0c5b
PB
2017 do {
2018 struct list_head rq_list;
2019 struct request *rq, *head_rq = list_entry_rq(list.next);
2020 struct list_head *pos = &head_rq->queuelist; /* skip first */
2021 struct blk_mq_hw_ctx *this_hctx = head_rq->mq_hctx;
2022 struct blk_mq_ctx *this_ctx = head_rq->mq_ctx;
2023 unsigned int depth = 1;
2024
2025 list_for_each_continue(pos, &list) {
2026 rq = list_entry_rq(pos);
2027 BUG_ON(!rq->q);
2028 if (rq->mq_hctx != this_hctx || rq->mq_ctx != this_ctx)
2029 break;
2030 depth++;
320ae51f
JA
2031 }
2032
95ed0c5b
PB
2033 list_cut_before(&rq_list, &list, pos);
2034 trace_block_unplug(head_rq->q, depth, !from_schedule);
67cae4c9 2035 blk_mq_sched_insert_requests(this_hctx, this_ctx, &rq_list,
bd166ef1 2036 from_schedule);
95ed0c5b 2037 } while(!list_empty(&list));
320ae51f
JA
2038}
2039
14ccb66b
CH
2040static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
2041 unsigned int nr_segs)
320ae51f 2042{
93f221ae
EB
2043 int err;
2044
f924cdde
CH
2045 if (bio->bi_opf & REQ_RAHEAD)
2046 rq->cmd_flags |= REQ_FAILFAST_MASK;
2047
2048 rq->__sector = bio->bi_iter.bi_sector;
2049 rq->write_hint = bio->bi_write_hint;
14ccb66b 2050 blk_rq_bio_prep(rq, bio, nr_segs);
93f221ae
EB
2051
2052 /* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
2053 err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
2054 WARN_ON_ONCE(err);
4b570521 2055
b5af37ab 2056 blk_account_io_start(rq);
320ae51f
JA
2057}
2058
0f95549c 2059static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
3e08773c 2060 struct request *rq, bool last)
f984df1f 2061{
f984df1f 2062 struct request_queue *q = rq->q;
f984df1f
SL
2063 struct blk_mq_queue_data bd = {
2064 .rq = rq,
be94f058 2065 .last = last,
f984df1f 2066 };
f06345ad 2067 blk_status_t ret;
0f95549c 2068
0f95549c
MS
2069 /*
2070 * For OK queue, we are done. For error, caller may kill it.
2071 * Any other error (busy), just add it to our list as we
2072 * previously would have done.
2073 */
2074 ret = q->mq_ops->queue_rq(hctx, &bd);
2075 switch (ret) {
2076 case BLK_STS_OK:
6ce3dd6e 2077 blk_mq_update_dispatch_busy(hctx, false);
0f95549c
MS
2078 break;
2079 case BLK_STS_RESOURCE:
86ff7c2a 2080 case BLK_STS_DEV_RESOURCE:
6ce3dd6e 2081 blk_mq_update_dispatch_busy(hctx, true);
0f95549c
MS
2082 __blk_mq_requeue_request(rq);
2083 break;
2084 default:
6ce3dd6e 2085 blk_mq_update_dispatch_busy(hctx, false);
0f95549c
MS
2086 break;
2087 }
2088
2089 return ret;
2090}
2091
fd9c40f6 2092static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
0f95549c 2093 struct request *rq,
fd9c40f6 2094 bool bypass_insert, bool last)
0f95549c
MS
2095{
2096 struct request_queue *q = rq->q;
d964f04a 2097 bool run_queue = true;
2a5a24aa 2098 int budget_token;
d964f04a 2099
23d4ee19 2100 /*
fd9c40f6 2101 * RCU or SRCU read lock is needed before checking quiesced flag.
23d4ee19 2102 *
fd9c40f6
BVA
2103 * When queue is stopped or quiesced, ignore 'bypass_insert' from
2104 * blk_mq_request_issue_directly(), and return BLK_STS_OK to caller,
2105 * and avoid driver to try to dispatch again.
23d4ee19 2106 */
fd9c40f6 2107 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) {
d964f04a 2108 run_queue = false;
fd9c40f6
BVA
2109 bypass_insert = false;
2110 goto insert;
d964f04a 2111 }
f984df1f 2112
fd9c40f6
BVA
2113 if (q->elevator && !bypass_insert)
2114 goto insert;
2253efc8 2115
2a5a24aa
ML
2116 budget_token = blk_mq_get_dispatch_budget(q);
2117 if (budget_token < 0)
fd9c40f6 2118 goto insert;
bd166ef1 2119
2a5a24aa
ML
2120 blk_mq_set_rq_budget_token(rq, budget_token);
2121
8ab6bb9e 2122 if (!blk_mq_get_driver_tag(rq)) {
2a5a24aa 2123 blk_mq_put_dispatch_budget(q, budget_token);
fd9c40f6 2124 goto insert;
88022d72 2125 }
de148297 2126
3e08773c 2127 return __blk_mq_issue_directly(hctx, rq, last);
fd9c40f6
BVA
2128insert:
2129 if (bypass_insert)
2130 return BLK_STS_RESOURCE;
2131
db03f88f
ML
2132 blk_mq_sched_insert_request(rq, false, run_queue, false);
2133
fd9c40f6
BVA
2134 return BLK_STS_OK;
2135}
2136
105663f7
AA
2137/**
2138 * blk_mq_try_issue_directly - Try to send a request directly to device driver.
2139 * @hctx: Pointer of the associated hardware queue.
2140 * @rq: Pointer to request to be sent.
105663f7
AA
2141 *
2142 * If the device has enough resources to accept a new request now, send the
2143 * request directly to device driver. Else, insert at hctx->dispatch queue, so
2144 * we can try send it another time in the future. Requests inserted at this
2145 * queue have higher priority.
2146 */
fd9c40f6 2147static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
3e08773c 2148 struct request *rq)
fd9c40f6
BVA
2149{
2150 blk_status_t ret;
2151 int srcu_idx;
2152
2153 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
2154
2155 hctx_lock(hctx, &srcu_idx);
2156
3e08773c 2157 ret = __blk_mq_try_issue_directly(hctx, rq, false, true);
fd9c40f6 2158 if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE)
01e99aec 2159 blk_mq_request_bypass_insert(rq, false, true);
fd9c40f6
BVA
2160 else if (ret != BLK_STS_OK)
2161 blk_mq_end_request(rq, ret);
2162
2163 hctx_unlock(hctx, srcu_idx);
2164}
2165
2166blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)
2167{
2168 blk_status_t ret;
2169 int srcu_idx;
fd9c40f6
BVA
2170 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2171
2172 hctx_lock(hctx, &srcu_idx);
3e08773c 2173 ret = __blk_mq_try_issue_directly(hctx, rq, true, last);
04ced159 2174 hctx_unlock(hctx, srcu_idx);
7f556a44
JW
2175
2176 return ret;
5eb6126e
CH
2177}
2178
6ce3dd6e
ML
2179void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
2180 struct list_head *list)
2181{
536167d4 2182 int queued = 0;
632bfb63 2183 int errors = 0;
536167d4 2184
6ce3dd6e 2185 while (!list_empty(list)) {
fd9c40f6 2186 blk_status_t ret;
6ce3dd6e
ML
2187 struct request *rq = list_first_entry(list, struct request,
2188 queuelist);
2189
2190 list_del_init(&rq->queuelist);
fd9c40f6
BVA
2191 ret = blk_mq_request_issue_directly(rq, list_empty(list));
2192 if (ret != BLK_STS_OK) {
2193 if (ret == BLK_STS_RESOURCE ||
2194 ret == BLK_STS_DEV_RESOURCE) {
01e99aec 2195 blk_mq_request_bypass_insert(rq, false,
c616cbee 2196 list_empty(list));
fd9c40f6
BVA
2197 break;
2198 }
2199 blk_mq_end_request(rq, ret);
632bfb63 2200 errors++;
536167d4
KB
2201 } else
2202 queued++;
6ce3dd6e 2203 }
d666ba98
JA
2204
2205 /*
2206 * If we didn't flush the entire list, we could have told
2207 * the driver there was more coming, but that turned out to
2208 * be a lie.
2209 */
632bfb63 2210 if ((!list_empty(list) || errors) &&
2211 hctx->queue->mq_ops->commit_rqs && queued)
d666ba98 2212 hctx->queue->mq_ops->commit_rqs(hctx);
6ce3dd6e
ML
2213}
2214
ce5b009c
JA
2215static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
2216{
2217 list_add_tail(&rq->queuelist, &plug->mq_list);
2218 plug->rq_count++;
2219 if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) {
2220 struct request *tmp;
2221
2222 tmp = list_first_entry(&plug->mq_list, struct request,
2223 queuelist);
2224 if (tmp->q != rq->q)
2225 plug->multiple_queues = true;
2226 }
2227}
2228
7f2a6a69 2229/*
ba0ffdd8 2230 * Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple
7f2a6a69
SL
2231 * queues. This is important for md arrays to benefit from merging
2232 * requests.
2233 */
2234static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug)
2235{
2236 if (plug->multiple_queues)
ba0ffdd8 2237 return BLK_MAX_REQUEST_COUNT * 2;
7f2a6a69
SL
2238 return BLK_MAX_REQUEST_COUNT;
2239}
2240
105663f7 2241/**
c62b37d9 2242 * blk_mq_submit_bio - Create and send a request to block device.
105663f7
AA
2243 * @bio: Bio pointer.
2244 *
2245 * Builds up a request structure from @q and @bio and send to the device. The
2246 * request may not be queued directly to hardware if:
2247 * * This request can be merged with another one
2248 * * We want to place request at plug queue for possible future merging
2249 * * There is an IO scheduler active at this queue
2250 *
2251 * It will not queue the request if there is an error with the bio, or at the
2252 * request creation.
105663f7 2253 */
3e08773c 2254void blk_mq_submit_bio(struct bio *bio)
07068d5b 2255{
ed6cddef 2256 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
ef295ecf 2257 const int is_sync = op_is_sync(bio->bi_opf);
f73f44eb 2258 const int is_flush_fua = op_is_flush(bio->bi_opf);
07068d5b 2259 struct request *rq;
f984df1f 2260 struct blk_plug *plug;
5b3f341f 2261 struct request *same_queue_rq = NULL;
abd45c15 2262 unsigned int nr_segs = 1;
a892c8d5 2263 blk_status_t ret;
07068d5b
JA
2264
2265 blk_queue_bounce(q, &bio);
abd45c15
JA
2266 if (blk_may_split(q, bio))
2267 __blk_queue_split(q, &bio, &nr_segs);
f36ea50c 2268
e23947bd 2269 if (!bio_integrity_prep(bio))
ac7c5675 2270 goto queue_exit;
07068d5b 2271
87c279e6 2272 if (!is_flush_fua && !blk_queue_nomerges(q) &&
14ccb66b 2273 blk_attempt_plug_merge(q, bio, nr_segs, &same_queue_rq))
ac7c5675 2274 goto queue_exit;
f984df1f 2275
14ccb66b 2276 if (blk_mq_sched_bio_merge(q, bio, nr_segs))
ac7c5675 2277 goto queue_exit;
bd166ef1 2278
d5337560 2279 rq_qos_throttle(q, bio);
87760e5e 2280
47c122e3
JA
2281 plug = blk_mq_plug(q, bio);
2282 if (plug && plug->cached_rq) {
2283 rq = plug->cached_rq;
2284 plug->cached_rq = rq->rq_next;
2285 INIT_LIST_HEAD(&rq->queuelist);
47c122e3 2286 } else {
0f38d766
CH
2287 struct blk_mq_alloc_data data = {
2288 .q = q,
2289 .nr_tags = 1,
2290 .cmd_flags = bio->bi_opf,
2291 };
2292
47c122e3
JA
2293 if (plug) {
2294 data.nr_tags = plug->nr_ios;
2295 plug->nr_ios = 1;
2296 data.cached_rq = &plug->cached_rq;
2297 }
b90cfaed 2298 rq = __blk_mq_alloc_requests(&data);
47c122e3
JA
2299 if (unlikely(!rq)) {
2300 rq_qos_cleanup(q, bio);
2301 if (bio->bi_opf & REQ_NOWAIT)
2302 bio_wouldblock_error(bio);
2303 goto queue_exit;
2304 }
87760e5e
JA
2305 }
2306
e8a676d6 2307 trace_block_getrq(bio);
d6f1dda2 2308
c1c80384 2309 rq_qos_track(q, rq, bio);
07068d5b 2310
970d168d
BVA
2311 blk_mq_bio_to_request(rq, bio, nr_segs);
2312
a892c8d5
ST
2313 ret = blk_crypto_init_request(rq);
2314 if (ret != BLK_STS_OK) {
2315 bio->bi_status = ret;
2316 bio_endio(bio);
2317 blk_mq_free_request(rq);
3e08773c 2318 return;
a892c8d5
ST
2319 }
2320
07068d5b 2321 if (unlikely(is_flush_fua)) {
4a60f360 2322 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
105663f7 2323 /* Bypass scheduler for flush requests */
923218f6 2324 blk_insert_flush(rq);
4a60f360 2325 blk_mq_run_hw_queue(hctx, true);
03f26d8f 2326 } else if (plug && (q->nr_hw_queues == 1 ||
079a2e3e 2327 blk_mq_is_shared_tags(rq->mq_hctx->flags) ||
03f26d8f 2328 q->mq_ops->commit_rqs || !blk_queue_nonrot(q))) {
b2c5d16b
JA
2329 /*
2330 * Use plugging if we have a ->commit_rqs() hook as well, as
2331 * we know the driver uses bd->last in a smart fashion.
3154df26
ML
2332 *
2333 * Use normal plugging if this disk is slow HDD, as sequential
2334 * IO may benefit a lot from plug merging.
b2c5d16b 2335 */
5f0ed774 2336 unsigned int request_count = plug->rq_count;
600271d9
SL
2337 struct request *last = NULL;
2338
676d0607 2339 if (!request_count)
e6c4438b 2340 trace_block_plug(q);
600271d9
SL
2341 else
2342 last = list_entry_rq(plug->mq_list.prev);
b094f89c 2343
7f2a6a69 2344 if (request_count >= blk_plug_max_rq_count(plug) || (last &&
600271d9 2345 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
e6c4438b
JM
2346 blk_flush_plug_list(plug, false);
2347 trace_block_plug(q);
320ae51f 2348 }
b094f89c 2349
ce5b009c 2350 blk_add_rq_to_plug(plug, rq);
a12de1d4 2351 } else if (q->elevator) {
105663f7 2352 /* Insert the request at the IO scheduler queue */
a12de1d4 2353 blk_mq_sched_insert_request(rq, false, true, true);
2299722c 2354 } else if (plug && !blk_queue_nomerges(q)) {
07068d5b 2355 /*
6a83e74d 2356 * We do limited plugging. If the bio can be merged, do that.
f984df1f
SL
2357 * Otherwise the existing request in the plug list will be
2358 * issued. So the plug list will have one request at most
2299722c
CH
2359 * The plug list might get flushed before this. If that happens,
2360 * the plug list is empty, and same_queue_rq is invalid.
07068d5b 2361 */
2299722c
CH
2362 if (list_empty(&plug->mq_list))
2363 same_queue_rq = NULL;
4711b573 2364 if (same_queue_rq) {
2299722c 2365 list_del_init(&same_queue_rq->queuelist);
4711b573
JA
2366 plug->rq_count--;
2367 }
ce5b009c 2368 blk_add_rq_to_plug(plug, rq);
ff3b74b8 2369 trace_block_plug(q);
2299722c 2370
dad7a3be 2371 if (same_queue_rq) {
ff3b74b8 2372 trace_block_unplug(q, 1, true);
0f38d766 2373 blk_mq_try_issue_directly(same_queue_rq->mq_hctx,
3e08773c 2374 same_queue_rq);
dad7a3be 2375 }
a12de1d4 2376 } else if ((q->nr_hw_queues > 1 && is_sync) ||
0f38d766 2377 !rq->mq_hctx->dispatch_busy) {
105663f7
AA
2378 /*
2379 * There is no scheduler and we can try to send directly
2380 * to the hardware.
2381 */
3e08773c 2382 blk_mq_try_issue_directly(rq->mq_hctx, rq);
ab42f35d 2383 } else {
105663f7 2384 /* Default case. */
8fa9f556 2385 blk_mq_sched_insert_request(rq, false, true, true);
ab42f35d 2386 }
320ae51f 2387
3e08773c 2388 return;
ac7c5675
CH
2389queue_exit:
2390 blk_queue_exit(q);
320ae51f
JA
2391}
2392
bd63141d
ML
2393static size_t order_to_size(unsigned int order)
2394{
2395 return (size_t)PAGE_SIZE << order;
2396}
2397
2398/* called before freeing request pool in @tags */
f32e4eaf
JG
2399static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags,
2400 struct blk_mq_tags *tags)
bd63141d 2401{
bd63141d
ML
2402 struct page *page;
2403 unsigned long flags;
2404
4f245d5b
JG
2405 /* There is no need to clear a driver tags own mapping */
2406 if (drv_tags == tags)
2407 return;
2408
bd63141d
ML
2409 list_for_each_entry(page, &tags->page_list, lru) {
2410 unsigned long start = (unsigned long)page_address(page);
2411 unsigned long end = start + order_to_size(page->private);
2412 int i;
2413
f32e4eaf 2414 for (i = 0; i < drv_tags->nr_tags; i++) {
bd63141d
ML
2415 struct request *rq = drv_tags->rqs[i];
2416 unsigned long rq_addr = (unsigned long)rq;
2417
2418 if (rq_addr >= start && rq_addr < end) {
2419 WARN_ON_ONCE(refcount_read(&rq->ref) != 0);
2420 cmpxchg(&drv_tags->rqs[i], rq, NULL);
2421 }
2422 }
2423 }
2424
2425 /*
2426 * Wait until all pending iteration is done.
2427 *
2428 * Request reference is cleared and it is guaranteed to be observed
2429 * after the ->lock is released.
2430 */
2431 spin_lock_irqsave(&drv_tags->lock, flags);
2432 spin_unlock_irqrestore(&drv_tags->lock, flags);
2433}
2434
cc71a6f4
JA
2435void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
2436 unsigned int hctx_idx)
95363efd 2437{
f32e4eaf 2438 struct blk_mq_tags *drv_tags;
e9b267d9 2439 struct page *page;
320ae51f 2440
079a2e3e
JG
2441 if (blk_mq_is_shared_tags(set->flags))
2442 drv_tags = set->shared_tags;
e155b0c2
JG
2443 else
2444 drv_tags = set->tags[hctx_idx];
f32e4eaf 2445
65de57bb 2446 if (tags->static_rqs && set->ops->exit_request) {
e9b267d9 2447 int i;
320ae51f 2448
24d2f903 2449 for (i = 0; i < tags->nr_tags; i++) {
2af8cbe3
JA
2450 struct request *rq = tags->static_rqs[i];
2451
2452 if (!rq)
e9b267d9 2453 continue;
d6296d39 2454 set->ops->exit_request(set, rq, hctx_idx);
2af8cbe3 2455 tags->static_rqs[i] = NULL;
e9b267d9 2456 }
320ae51f 2457 }
320ae51f 2458
f32e4eaf 2459 blk_mq_clear_rq_mapping(drv_tags, tags);
bd63141d 2460
24d2f903
CH
2461 while (!list_empty(&tags->page_list)) {
2462 page = list_first_entry(&tags->page_list, struct page, lru);
6753471c 2463 list_del_init(&page->lru);
f75782e4
CM
2464 /*
2465 * Remove kmemleak object previously allocated in
273938bf 2466 * blk_mq_alloc_rqs().
f75782e4
CM
2467 */
2468 kmemleak_free(page_address(page));
320ae51f
JA
2469 __free_pages(page, page->private);
2470 }
cc71a6f4 2471}
320ae51f 2472
e155b0c2 2473void blk_mq_free_rq_map(struct blk_mq_tags *tags)
cc71a6f4 2474{
24d2f903 2475 kfree(tags->rqs);
cc71a6f4 2476 tags->rqs = NULL;
2af8cbe3
JA
2477 kfree(tags->static_rqs);
2478 tags->static_rqs = NULL;
320ae51f 2479
e155b0c2 2480 blk_mq_free_tags(tags);
320ae51f
JA
2481}
2482
63064be1
JG
2483static struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
2484 unsigned int hctx_idx,
2485 unsigned int nr_tags,
e155b0c2 2486 unsigned int reserved_tags)
320ae51f 2487{
24d2f903 2488 struct blk_mq_tags *tags;
59f082e4 2489 int node;
320ae51f 2490
7d76f856 2491 node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx);
59f082e4
SL
2492 if (node == NUMA_NO_NODE)
2493 node = set->numa_node;
2494
e155b0c2
JG
2495 tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
2496 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
24d2f903
CH
2497 if (!tags)
2498 return NULL;
320ae51f 2499
590b5b7d 2500 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
36e1f3d1 2501 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
59f082e4 2502 node);
24d2f903 2503 if (!tags->rqs) {
e155b0c2 2504 blk_mq_free_tags(tags);
24d2f903
CH
2505 return NULL;
2506 }
320ae51f 2507
590b5b7d
KC
2508 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
2509 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
2510 node);
2af8cbe3
JA
2511 if (!tags->static_rqs) {
2512 kfree(tags->rqs);
e155b0c2 2513 blk_mq_free_tags(tags);
2af8cbe3
JA
2514 return NULL;
2515 }
2516
cc71a6f4
JA
2517 return tags;
2518}
2519
1d9bd516
TH
2520static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
2521 unsigned int hctx_idx, int node)
2522{
2523 int ret;
2524
2525 if (set->ops->init_request) {
2526 ret = set->ops->init_request(set, rq, hctx_idx, node);
2527 if (ret)
2528 return ret;
2529 }
2530
12f5b931 2531 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
1d9bd516
TH
2532 return 0;
2533}
2534
63064be1
JG
2535static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set,
2536 struct blk_mq_tags *tags,
2537 unsigned int hctx_idx, unsigned int depth)
cc71a6f4
JA
2538{
2539 unsigned int i, j, entries_per_page, max_order = 4;
2540 size_t rq_size, left;
59f082e4
SL
2541 int node;
2542
7d76f856 2543 node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx);
59f082e4
SL
2544 if (node == NUMA_NO_NODE)
2545 node = set->numa_node;
cc71a6f4
JA
2546
2547 INIT_LIST_HEAD(&tags->page_list);
2548
320ae51f
JA
2549 /*
2550 * rq_size is the size of the request plus driver payload, rounded
2551 * to the cacheline size
2552 */
24d2f903 2553 rq_size = round_up(sizeof(struct request) + set->cmd_size,
320ae51f 2554 cache_line_size());
cc71a6f4 2555 left = rq_size * depth;
320ae51f 2556
cc71a6f4 2557 for (i = 0; i < depth; ) {
320ae51f
JA
2558 int this_order = max_order;
2559 struct page *page;
2560 int to_do;
2561 void *p;
2562
b3a834b1 2563 while (this_order && left < order_to_size(this_order - 1))
320ae51f
JA
2564 this_order--;
2565
2566 do {
59f082e4 2567 page = alloc_pages_node(node,
36e1f3d1 2568 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
a5164405 2569 this_order);
320ae51f
JA
2570 if (page)
2571 break;
2572 if (!this_order--)
2573 break;
2574 if (order_to_size(this_order) < rq_size)
2575 break;
2576 } while (1);
2577
2578 if (!page)
24d2f903 2579 goto fail;
320ae51f
JA
2580
2581 page->private = this_order;
24d2f903 2582 list_add_tail(&page->lru, &tags->page_list);
320ae51f
JA
2583
2584 p = page_address(page);
f75782e4
CM
2585 /*
2586 * Allow kmemleak to scan these pages as they contain pointers
2587 * to additional allocations like via ops->init_request().
2588 */
36e1f3d1 2589 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
320ae51f 2590 entries_per_page = order_to_size(this_order) / rq_size;
cc71a6f4 2591 to_do = min(entries_per_page, depth - i);
320ae51f
JA
2592 left -= to_do * rq_size;
2593 for (j = 0; j < to_do; j++) {
2af8cbe3
JA
2594 struct request *rq = p;
2595
2596 tags->static_rqs[i] = rq;
1d9bd516
TH
2597 if (blk_mq_init_request(set, rq, hctx_idx, node)) {
2598 tags->static_rqs[i] = NULL;
2599 goto fail;
e9b267d9
CH
2600 }
2601
320ae51f
JA
2602 p += rq_size;
2603 i++;
2604 }
2605 }
cc71a6f4 2606 return 0;
320ae51f 2607
24d2f903 2608fail:
cc71a6f4
JA
2609 blk_mq_free_rqs(set, tags, hctx_idx);
2610 return -ENOMEM;
320ae51f
JA
2611}
2612
bf0beec0
ML
2613struct rq_iter_data {
2614 struct blk_mq_hw_ctx *hctx;
2615 bool has_rq;
2616};
2617
2618static bool blk_mq_has_request(struct request *rq, void *data, bool reserved)
2619{
2620 struct rq_iter_data *iter_data = data;
2621
2622 if (rq->mq_hctx != iter_data->hctx)
2623 return true;
2624 iter_data->has_rq = true;
2625 return false;
2626}
2627
2628static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
2629{
2630 struct blk_mq_tags *tags = hctx->sched_tags ?
2631 hctx->sched_tags : hctx->tags;
2632 struct rq_iter_data data = {
2633 .hctx = hctx,
2634 };
2635
2636 blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
2637 return data.has_rq;
2638}
2639
2640static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu,
2641 struct blk_mq_hw_ctx *hctx)
2642{
2643 if (cpumask_next_and(-1, hctx->cpumask, cpu_online_mask) != cpu)
2644 return false;
2645 if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids)
2646 return false;
2647 return true;
2648}
2649
2650static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
2651{
2652 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
2653 struct blk_mq_hw_ctx, cpuhp_online);
2654
2655 if (!cpumask_test_cpu(cpu, hctx->cpumask) ||
2656 !blk_mq_last_cpu_in_hctx(cpu, hctx))
2657 return 0;
2658
2659 /*
2660 * Prevent new request from being allocated on the current hctx.
2661 *
2662 * The smp_mb__after_atomic() Pairs with the implied barrier in
2663 * test_and_set_bit_lock in sbitmap_get(). Ensures the inactive flag is
2664 * seen once we return from the tag allocator.
2665 */
2666 set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
2667 smp_mb__after_atomic();
2668
2669 /*
2670 * Try to grab a reference to the queue and wait for any outstanding
2671 * requests. If we could not grab a reference the queue has been
2672 * frozen and there are no requests.
2673 */
2674 if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
2675 while (blk_mq_hctx_has_requests(hctx))
2676 msleep(5);
2677 percpu_ref_put(&hctx->queue->q_usage_counter);
2678 }
2679
2680 return 0;
2681}
2682
2683static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
2684{
2685 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
2686 struct blk_mq_hw_ctx, cpuhp_online);
2687
2688 if (cpumask_test_cpu(cpu, hctx->cpumask))
2689 clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
2690 return 0;
2691}
2692
e57690fe
JA
2693/*
2694 * 'cpu' is going away. splice any existing rq_list entries from this
2695 * software queue to the hw queue dispatch list, and ensure that it
2696 * gets run.
2697 */
9467f859 2698static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
484b4061 2699{
9467f859 2700 struct blk_mq_hw_ctx *hctx;
484b4061
JA
2701 struct blk_mq_ctx *ctx;
2702 LIST_HEAD(tmp);
c16d6b5a 2703 enum hctx_type type;
484b4061 2704
9467f859 2705 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
bf0beec0
ML
2706 if (!cpumask_test_cpu(cpu, hctx->cpumask))
2707 return 0;
2708
e57690fe 2709 ctx = __blk_mq_get_ctx(hctx->queue, cpu);
c16d6b5a 2710 type = hctx->type;
484b4061
JA
2711
2712 spin_lock(&ctx->lock);
c16d6b5a
ML
2713 if (!list_empty(&ctx->rq_lists[type])) {
2714 list_splice_init(&ctx->rq_lists[type], &tmp);
484b4061
JA
2715 blk_mq_hctx_clear_pending(hctx, ctx);
2716 }
2717 spin_unlock(&ctx->lock);
2718
2719 if (list_empty(&tmp))
9467f859 2720 return 0;
484b4061 2721
e57690fe
JA
2722 spin_lock(&hctx->lock);
2723 list_splice_tail_init(&tmp, &hctx->dispatch);
2724 spin_unlock(&hctx->lock);
484b4061
JA
2725
2726 blk_mq_run_hw_queue(hctx, true);
9467f859 2727 return 0;
484b4061
JA
2728}
2729
9467f859 2730static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
484b4061 2731{
bf0beec0
ML
2732 if (!(hctx->flags & BLK_MQ_F_STACKING))
2733 cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
2734 &hctx->cpuhp_online);
9467f859
TG
2735 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
2736 &hctx->cpuhp_dead);
484b4061
JA
2737}
2738
364b6181
ML
2739/*
2740 * Before freeing hw queue, clearing the flush request reference in
2741 * tags->rqs[] for avoiding potential UAF.
2742 */
2743static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags,
2744 unsigned int queue_depth, struct request *flush_rq)
2745{
2746 int i;
2747 unsigned long flags;
2748
2749 /* The hw queue may not be mapped yet */
2750 if (!tags)
2751 return;
2752
2753 WARN_ON_ONCE(refcount_read(&flush_rq->ref) != 0);
2754
2755 for (i = 0; i < queue_depth; i++)
2756 cmpxchg(&tags->rqs[i], flush_rq, NULL);
2757
2758 /*
2759 * Wait until all pending iteration is done.
2760 *
2761 * Request reference is cleared and it is guaranteed to be observed
2762 * after the ->lock is released.
2763 */
2764 spin_lock_irqsave(&tags->lock, flags);
2765 spin_unlock_irqrestore(&tags->lock, flags);
2766}
2767
c3b4afca 2768/* hctx->ctxs will be freed in queue's release handler */
08e98fc6
ML
2769static void blk_mq_exit_hctx(struct request_queue *q,
2770 struct blk_mq_tag_set *set,
2771 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
2772{
364b6181
ML
2773 struct request *flush_rq = hctx->fq->flush_rq;
2774
8ab0b7dc
ML
2775 if (blk_mq_hw_queue_mapped(hctx))
2776 blk_mq_tag_idle(hctx);
08e98fc6 2777
364b6181
ML
2778 blk_mq_clear_flush_rq_mapping(set->tags[hctx_idx],
2779 set->queue_depth, flush_rq);
f70ced09 2780 if (set->ops->exit_request)
364b6181 2781 set->ops->exit_request(set, flush_rq, hctx_idx);
f70ced09 2782
08e98fc6
ML
2783 if (set->ops->exit_hctx)
2784 set->ops->exit_hctx(hctx, hctx_idx);
2785
9467f859 2786 blk_mq_remove_cpuhp(hctx);
2f8f1336
ML
2787
2788 spin_lock(&q->unused_hctx_lock);
2789 list_add(&hctx->hctx_list, &q->unused_hctx_list);
2790 spin_unlock(&q->unused_hctx_lock);
08e98fc6
ML
2791}
2792
624dbe47
ML
2793static void blk_mq_exit_hw_queues(struct request_queue *q,
2794 struct blk_mq_tag_set *set, int nr_queue)
2795{
2796 struct blk_mq_hw_ctx *hctx;
2797 unsigned int i;
2798
2799 queue_for_each_hw_ctx(q, hctx, i) {
2800 if (i == nr_queue)
2801 break;
477e19de 2802 blk_mq_debugfs_unregister_hctx(hctx);
08e98fc6 2803 blk_mq_exit_hctx(q, set, hctx, i);
624dbe47 2804 }
624dbe47
ML
2805}
2806
7c6c5b7c
ML
2807static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set)
2808{
2809 int hw_ctx_size = sizeof(struct blk_mq_hw_ctx);
2810
2811 BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, srcu),
2812 __alignof__(struct blk_mq_hw_ctx)) !=
2813 sizeof(struct blk_mq_hw_ctx));
2814
2815 if (tag_set->flags & BLK_MQ_F_BLOCKING)
2816 hw_ctx_size += sizeof(struct srcu_struct);
2817
2818 return hw_ctx_size;
2819}
2820
08e98fc6
ML
2821static int blk_mq_init_hctx(struct request_queue *q,
2822 struct blk_mq_tag_set *set,
2823 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
320ae51f 2824{
7c6c5b7c
ML
2825 hctx->queue_num = hctx_idx;
2826
bf0beec0
ML
2827 if (!(hctx->flags & BLK_MQ_F_STACKING))
2828 cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
2829 &hctx->cpuhp_online);
7c6c5b7c
ML
2830 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
2831
2832 hctx->tags = set->tags[hctx_idx];
2833
2834 if (set->ops->init_hctx &&
2835 set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
2836 goto unregister_cpu_notifier;
08e98fc6 2837
7c6c5b7c
ML
2838 if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx,
2839 hctx->numa_node))
2840 goto exit_hctx;
2841 return 0;
2842
2843 exit_hctx:
2844 if (set->ops->exit_hctx)
2845 set->ops->exit_hctx(hctx, hctx_idx);
2846 unregister_cpu_notifier:
2847 blk_mq_remove_cpuhp(hctx);
2848 return -1;
2849}
2850
2851static struct blk_mq_hw_ctx *
2852blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
2853 int node)
2854{
2855 struct blk_mq_hw_ctx *hctx;
2856 gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;
2857
2858 hctx = kzalloc_node(blk_mq_hw_ctx_size(set), gfp, node);
2859 if (!hctx)
2860 goto fail_alloc_hctx;
2861
2862 if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
2863 goto free_hctx;
2864
2865 atomic_set(&hctx->nr_active, 0);
08e98fc6 2866 if (node == NUMA_NO_NODE)
7c6c5b7c
ML
2867 node = set->numa_node;
2868 hctx->numa_node = node;
08e98fc6 2869
9f993737 2870 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
08e98fc6
ML
2871 spin_lock_init(&hctx->lock);
2872 INIT_LIST_HEAD(&hctx->dispatch);
2873 hctx->queue = q;
51db1c37 2874 hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
08e98fc6 2875
2f8f1336
ML
2876 INIT_LIST_HEAD(&hctx->hctx_list);
2877
320ae51f 2878 /*
08e98fc6
ML
2879 * Allocate space for all possible cpus to avoid allocation at
2880 * runtime
320ae51f 2881 */
d904bfa7 2882 hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),
7c6c5b7c 2883 gfp, node);
08e98fc6 2884 if (!hctx->ctxs)
7c6c5b7c 2885 goto free_cpumask;
320ae51f 2886
5b202853 2887 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
c548e62b 2888 gfp, node, false, false))
08e98fc6 2889 goto free_ctxs;
08e98fc6 2890 hctx->nr_ctx = 0;
320ae51f 2891
5815839b 2892 spin_lock_init(&hctx->dispatch_wait_lock);
eb619fdb
JA
2893 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
2894 INIT_LIST_HEAD(&hctx->dispatch_wait.entry);
2895
754a1572 2896 hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp);
f70ced09 2897 if (!hctx->fq)
7c6c5b7c 2898 goto free_bitmap;
320ae51f 2899
6a83e74d 2900 if (hctx->flags & BLK_MQ_F_BLOCKING)
05707b64 2901 init_srcu_struct(hctx->srcu);
7c6c5b7c 2902 blk_mq_hctx_kobj_init(hctx);
6a83e74d 2903
7c6c5b7c 2904 return hctx;
320ae51f 2905
08e98fc6 2906 free_bitmap:
88459642 2907 sbitmap_free(&hctx->ctx_map);
08e98fc6
ML
2908 free_ctxs:
2909 kfree(hctx->ctxs);
7c6c5b7c
ML
2910 free_cpumask:
2911 free_cpumask_var(hctx->cpumask);
2912 free_hctx:
2913 kfree(hctx);
2914 fail_alloc_hctx:
2915 return NULL;
08e98fc6 2916}
320ae51f 2917
320ae51f
JA
2918static void blk_mq_init_cpu_queues(struct request_queue *q,
2919 unsigned int nr_hw_queues)
2920{
b3c661b1
JA
2921 struct blk_mq_tag_set *set = q->tag_set;
2922 unsigned int i, j;
320ae51f
JA
2923
2924 for_each_possible_cpu(i) {
2925 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
2926 struct blk_mq_hw_ctx *hctx;
c16d6b5a 2927 int k;
320ae51f 2928
320ae51f
JA
2929 __ctx->cpu = i;
2930 spin_lock_init(&__ctx->lock);
c16d6b5a
ML
2931 for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++)
2932 INIT_LIST_HEAD(&__ctx->rq_lists[k]);
2933
320ae51f
JA
2934 __ctx->queue = q;
2935
320ae51f
JA
2936 /*
2937 * Set local node, IFF we have more than one hw queue. If
2938 * not, we remain on the home node of the device
2939 */
b3c661b1
JA
2940 for (j = 0; j < set->nr_maps; j++) {
2941 hctx = blk_mq_map_queue_type(q, j, i);
2942 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
576e85c5 2943 hctx->numa_node = cpu_to_node(i);
b3c661b1 2944 }
320ae51f
JA
2945 }
2946}
2947
63064be1
JG
2948struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
2949 unsigned int hctx_idx,
2950 unsigned int depth)
cc71a6f4 2951{
63064be1
JG
2952 struct blk_mq_tags *tags;
2953 int ret;
cc71a6f4 2954
e155b0c2 2955 tags = blk_mq_alloc_rq_map(set, hctx_idx, depth, set->reserved_tags);
63064be1
JG
2956 if (!tags)
2957 return NULL;
cc71a6f4 2958
63064be1
JG
2959 ret = blk_mq_alloc_rqs(set, tags, hctx_idx, depth);
2960 if (ret) {
e155b0c2 2961 blk_mq_free_rq_map(tags);
63064be1
JG
2962 return NULL;
2963 }
cc71a6f4 2964
63064be1
JG
2965 return tags;
2966}
2967
2968static bool __blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
2969 int hctx_idx)
2970{
079a2e3e
JG
2971 if (blk_mq_is_shared_tags(set->flags)) {
2972 set->tags[hctx_idx] = set->shared_tags;
e155b0c2
JG
2973
2974 return true;
2975 }
2976
63064be1
JG
2977 set->tags[hctx_idx] = blk_mq_alloc_map_and_rqs(set, hctx_idx,
2978 set->queue_depth);
2979
2980 return set->tags[hctx_idx];
cc71a6f4
JA
2981}
2982
645db34e
JG
2983void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
2984 struct blk_mq_tags *tags,
2985 unsigned int hctx_idx)
cc71a6f4 2986{
645db34e
JG
2987 if (tags) {
2988 blk_mq_free_rqs(set, tags, hctx_idx);
e155b0c2 2989 blk_mq_free_rq_map(tags);
bd166ef1 2990 }
cc71a6f4
JA
2991}
2992
e155b0c2
JG
2993static void __blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
2994 unsigned int hctx_idx)
2995{
079a2e3e 2996 if (!blk_mq_is_shared_tags(set->flags))
e155b0c2
JG
2997 blk_mq_free_map_and_rqs(set, set->tags[hctx_idx], hctx_idx);
2998
2999 set->tags[hctx_idx] = NULL;
3000}
3001
4b855ad3 3002static void blk_mq_map_swqueue(struct request_queue *q)
320ae51f 3003{
b3c661b1 3004 unsigned int i, j, hctx_idx;
320ae51f
JA
3005 struct blk_mq_hw_ctx *hctx;
3006 struct blk_mq_ctx *ctx;
2a34c087 3007 struct blk_mq_tag_set *set = q->tag_set;
320ae51f
JA
3008
3009 queue_for_each_hw_ctx(q, hctx, i) {
e4043dcf 3010 cpumask_clear(hctx->cpumask);
320ae51f 3011 hctx->nr_ctx = 0;
d416c92c 3012 hctx->dispatch_from = NULL;
320ae51f
JA
3013 }
3014
3015 /*
4b855ad3 3016 * Map software to hardware queues.
4412efec
ML
3017 *
3018 * If the cpu isn't present, the cpu is mapped to first hctx.
320ae51f 3019 */
20e4d813 3020 for_each_possible_cpu(i) {
4412efec 3021
897bb0c7 3022 ctx = per_cpu_ptr(q->queue_ctx, i);
b3c661b1 3023 for (j = 0; j < set->nr_maps; j++) {
bb94aea1
JW
3024 if (!set->map[j].nr_queues) {
3025 ctx->hctxs[j] = blk_mq_map_queue_type(q,
3026 HCTX_TYPE_DEFAULT, i);
e5edd5f2 3027 continue;
bb94aea1 3028 }
fd689871
ML
3029 hctx_idx = set->map[j].mq_map[i];
3030 /* unmapped hw queue can be remapped after CPU topo changed */
3031 if (!set->tags[hctx_idx] &&
63064be1 3032 !__blk_mq_alloc_map_and_rqs(set, hctx_idx)) {
fd689871
ML
3033 /*
3034 * If tags initialization fail for some hctx,
3035 * that hctx won't be brought online. In this
3036 * case, remap the current ctx to hctx[0] which
3037 * is guaranteed to always have tags allocated
3038 */
3039 set->map[j].mq_map[i] = 0;
3040 }
e5edd5f2 3041
b3c661b1 3042 hctx = blk_mq_map_queue_type(q, j, i);
8ccdf4a3 3043 ctx->hctxs[j] = hctx;
b3c661b1
JA
3044 /*
3045 * If the CPU is already set in the mask, then we've
3046 * mapped this one already. This can happen if
3047 * devices share queues across queue maps.
3048 */
3049 if (cpumask_test_cpu(i, hctx->cpumask))
3050 continue;
3051
3052 cpumask_set_cpu(i, hctx->cpumask);
3053 hctx->type = j;
3054 ctx->index_hw[hctx->type] = hctx->nr_ctx;
3055 hctx->ctxs[hctx->nr_ctx++] = ctx;
3056
3057 /*
3058 * If the nr_ctx type overflows, we have exceeded the
3059 * amount of sw queues we can support.
3060 */
3061 BUG_ON(!hctx->nr_ctx);
3062 }
bb94aea1
JW
3063
3064 for (; j < HCTX_MAX_TYPES; j++)
3065 ctx->hctxs[j] = blk_mq_map_queue_type(q,
3066 HCTX_TYPE_DEFAULT, i);
320ae51f 3067 }
506e931f
JA
3068
3069 queue_for_each_hw_ctx(q, hctx, i) {
4412efec
ML
3070 /*
3071 * If no software queues are mapped to this hardware queue,
3072 * disable it and free the request entries.
3073 */
3074 if (!hctx->nr_ctx) {
3075 /* Never unmap queue 0. We need it as a
3076 * fallback in case of a new remap fails
3077 * allocation
3078 */
e155b0c2
JG
3079 if (i)
3080 __blk_mq_free_map_and_rqs(set, i);
4412efec
ML
3081
3082 hctx->tags = NULL;
3083 continue;
3084 }
484b4061 3085
2a34c087
ML
3086 hctx->tags = set->tags[i];
3087 WARN_ON(!hctx->tags);
3088
889fa31f
CY
3089 /*
3090 * Set the map size to the number of mapped software queues.
3091 * This is more accurate and more efficient than looping
3092 * over all possibly mapped software queues.
3093 */
88459642 3094 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
889fa31f 3095
484b4061
JA
3096 /*
3097 * Initialize batch roundrobin counts
3098 */
f82ddf19 3099 hctx->next_cpu = blk_mq_first_mapped_cpu(hctx);
506e931f
JA
3100 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
3101 }
320ae51f
JA
3102}
3103
8e8320c9
JA
3104/*
3105 * Caller needs to ensure that we're either frozen/quiesced, or that
3106 * the queue isn't live yet.
3107 */
2404e607 3108static void queue_set_hctx_shared(struct request_queue *q, bool shared)
0d2602ca
JA
3109{
3110 struct blk_mq_hw_ctx *hctx;
0d2602ca
JA
3111 int i;
3112
2404e607 3113 queue_for_each_hw_ctx(q, hctx, i) {
454bb677 3114 if (shared) {
51db1c37 3115 hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
454bb677
YK
3116 } else {
3117 blk_mq_tag_idle(hctx);
51db1c37 3118 hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
454bb677 3119 }
2404e607
JM
3120 }
3121}
3122
655ac300
HR
3123static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
3124 bool shared)
2404e607
JM
3125{
3126 struct request_queue *q;
0d2602ca 3127
705cda97
BVA
3128 lockdep_assert_held(&set->tag_list_lock);
3129
0d2602ca
JA
3130 list_for_each_entry(q, &set->tag_list, tag_set_list) {
3131 blk_mq_freeze_queue(q);
2404e607 3132 queue_set_hctx_shared(q, shared);
0d2602ca
JA
3133 blk_mq_unfreeze_queue(q);
3134 }
3135}
3136
3137static void blk_mq_del_queue_tag_set(struct request_queue *q)
3138{
3139 struct blk_mq_tag_set *set = q->tag_set;
3140
0d2602ca 3141 mutex_lock(&set->tag_list_lock);
08c875cb 3142 list_del(&q->tag_set_list);
2404e607
JM
3143 if (list_is_singular(&set->tag_list)) {
3144 /* just transitioned to unshared */
51db1c37 3145 set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
2404e607 3146 /* update existing queue */
655ac300 3147 blk_mq_update_tag_set_shared(set, false);
2404e607 3148 }
0d2602ca 3149 mutex_unlock(&set->tag_list_lock);
a347c7ad 3150 INIT_LIST_HEAD(&q->tag_set_list);
0d2602ca
JA
3151}
3152
3153static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
3154 struct request_queue *q)
3155{
0d2602ca 3156 mutex_lock(&set->tag_list_lock);
2404e607 3157
ff821d27
JA
3158 /*
3159 * Check to see if we're transitioning to shared (from 1 to 2 queues).
3160 */
3161 if (!list_empty(&set->tag_list) &&
51db1c37
ML
3162 !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
3163 set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
2404e607 3164 /* update existing queue */
655ac300 3165 blk_mq_update_tag_set_shared(set, true);
2404e607 3166 }
51db1c37 3167 if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
2404e607 3168 queue_set_hctx_shared(q, true);
08c875cb 3169 list_add_tail(&q->tag_set_list, &set->tag_list);
2404e607 3170
0d2602ca
JA
3171 mutex_unlock(&set->tag_list_lock);
3172}
3173
1db4909e
ML
3174/* All allocations will be freed in release handler of q->mq_kobj */
3175static int blk_mq_alloc_ctxs(struct request_queue *q)
3176{
3177 struct blk_mq_ctxs *ctxs;
3178 int cpu;
3179
3180 ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL);
3181 if (!ctxs)
3182 return -ENOMEM;
3183
3184 ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx);
3185 if (!ctxs->queue_ctx)
3186 goto fail;
3187
3188 for_each_possible_cpu(cpu) {
3189 struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu);
3190 ctx->ctxs = ctxs;
3191 }
3192
3193 q->mq_kobj = &ctxs->kobj;
3194 q->queue_ctx = ctxs->queue_ctx;
3195
3196 return 0;
3197 fail:
3198 kfree(ctxs);
3199 return -ENOMEM;
3200}
3201
e09aae7e
ML
3202/*
3203 * It is the actual release handler for mq, but we do it from
3204 * request queue's release handler for avoiding use-after-free
3205 * and headache because q->mq_kobj shouldn't have been introduced,
3206 * but we can't group ctx/kctx kobj without it.
3207 */
3208void blk_mq_release(struct request_queue *q)
3209{
2f8f1336
ML
3210 struct blk_mq_hw_ctx *hctx, *next;
3211 int i;
e09aae7e 3212
2f8f1336
ML
3213 queue_for_each_hw_ctx(q, hctx, i)
3214 WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));
3215
3216 /* all hctx are in .unused_hctx_list now */
3217 list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
3218 list_del_init(&hctx->hctx_list);
6c8b232e 3219 kobject_put(&hctx->kobj);
c3b4afca 3220 }
e09aae7e
ML
3221
3222 kfree(q->queue_hw_ctx);
3223
7ea5fe31
ML
3224 /*
3225 * release .mq_kobj and sw queue's kobject now because
3226 * both share lifetime with request queue.
3227 */
3228 blk_mq_sysfs_deinit(q);
e09aae7e
ML
3229}
3230
5ec780a6 3231static struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set,
2f227bb9 3232 void *queuedata)
b62c21b7 3233{
26a9750a
CH
3234 struct request_queue *q;
3235 int ret;
b62c21b7 3236
26a9750a
CH
3237 q = blk_alloc_queue(set->numa_node);
3238 if (!q)
b62c21b7 3239 return ERR_PTR(-ENOMEM);
26a9750a
CH
3240 q->queuedata = queuedata;
3241 ret = blk_mq_init_allocated_queue(set, q);
3242 if (ret) {
3243 blk_cleanup_queue(q);
3244 return ERR_PTR(ret);
3245 }
b62c21b7
MS
3246 return q;
3247}
2f227bb9
CH
3248
3249struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
3250{
3251 return blk_mq_init_queue_data(set, NULL);
3252}
b62c21b7
MS
3253EXPORT_SYMBOL(blk_mq_init_queue);
3254
4dcc4874
CH
3255struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata,
3256 struct lock_class_key *lkclass)
9316a9ed
JA
3257{
3258 struct request_queue *q;
b461dfc4 3259 struct gendisk *disk;
9316a9ed 3260
b461dfc4
CH
3261 q = blk_mq_init_queue_data(set, queuedata);
3262 if (IS_ERR(q))
3263 return ERR_CAST(q);
9316a9ed 3264
4a1fa41d 3265 disk = __alloc_disk_node(q, set->numa_node, lkclass);
b461dfc4
CH
3266 if (!disk) {
3267 blk_cleanup_queue(q);
3268 return ERR_PTR(-ENOMEM);
9316a9ed 3269 }
b461dfc4 3270 return disk;
9316a9ed 3271}
b461dfc4 3272EXPORT_SYMBOL(__blk_mq_alloc_disk);
9316a9ed 3273
34d11ffa
JW
3274static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
3275 struct blk_mq_tag_set *set, struct request_queue *q,
3276 int hctx_idx, int node)
3277{
2f8f1336 3278 struct blk_mq_hw_ctx *hctx = NULL, *tmp;
34d11ffa 3279
2f8f1336
ML
3280 /* reuse dead hctx first */
3281 spin_lock(&q->unused_hctx_lock);
3282 list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) {
3283 if (tmp->numa_node == node) {
3284 hctx = tmp;
3285 break;
3286 }
3287 }
3288 if (hctx)
3289 list_del_init(&hctx->hctx_list);
3290 spin_unlock(&q->unused_hctx_lock);
3291
3292 if (!hctx)
3293 hctx = blk_mq_alloc_hctx(q, set, node);
34d11ffa 3294 if (!hctx)
7c6c5b7c 3295 goto fail;
34d11ffa 3296
7c6c5b7c
ML
3297 if (blk_mq_init_hctx(q, set, hctx, hctx_idx))
3298 goto free_hctx;
34d11ffa
JW
3299
3300 return hctx;
7c6c5b7c
ML
3301
3302 free_hctx:
3303 kobject_put(&hctx->kobj);
3304 fail:
3305 return NULL;
34d11ffa
JW
3306}
3307
868f2f0b
KB
3308static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
3309 struct request_queue *q)
320ae51f 3310{
e01ad46d 3311 int i, j, end;
868f2f0b 3312 struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
f14bbe77 3313
ac0d6b92
BVA
3314 if (q->nr_hw_queues < set->nr_hw_queues) {
3315 struct blk_mq_hw_ctx **new_hctxs;
3316
3317 new_hctxs = kcalloc_node(set->nr_hw_queues,
3318 sizeof(*new_hctxs), GFP_KERNEL,
3319 set->numa_node);
3320 if (!new_hctxs)
3321 return;
3322 if (hctxs)
3323 memcpy(new_hctxs, hctxs, q->nr_hw_queues *
3324 sizeof(*hctxs));
3325 q->queue_hw_ctx = new_hctxs;
ac0d6b92
BVA
3326 kfree(hctxs);
3327 hctxs = new_hctxs;
3328 }
3329
fb350e0a
ML
3330 /* protect against switching io scheduler */
3331 mutex_lock(&q->sysfs_lock);
24d2f903 3332 for (i = 0; i < set->nr_hw_queues; i++) {
868f2f0b 3333 int node;
34d11ffa 3334 struct blk_mq_hw_ctx *hctx;
868f2f0b 3335
7d76f856 3336 node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], i);
34d11ffa
JW
3337 /*
3338 * If the hw queue has been mapped to another numa node,
3339 * we need to realloc the hctx. If allocation fails, fallback
3340 * to use the previous one.
3341 */
3342 if (hctxs[i] && (hctxs[i]->numa_node == node))
3343 continue;
868f2f0b 3344
34d11ffa
JW
3345 hctx = blk_mq_alloc_and_init_hctx(set, q, i, node);
3346 if (hctx) {
2f8f1336 3347 if (hctxs[i])
34d11ffa 3348 blk_mq_exit_hctx(q, set, hctxs[i], i);
34d11ffa
JW
3349 hctxs[i] = hctx;
3350 } else {
3351 if (hctxs[i])
3352 pr_warn("Allocate new hctx on node %d fails,\
3353 fallback to previous one on node %d\n",
3354 node, hctxs[i]->numa_node);
3355 else
3356 break;
868f2f0b 3357 }
320ae51f 3358 }
e01ad46d
JW
3359 /*
3360 * Increasing nr_hw_queues fails. Free the newly allocated
3361 * hctxs and keep the previous q->nr_hw_queues.
3362 */
3363 if (i != set->nr_hw_queues) {
3364 j = q->nr_hw_queues;
3365 end = i;
3366 } else {
3367 j = i;
3368 end = q->nr_hw_queues;
3369 q->nr_hw_queues = set->nr_hw_queues;
3370 }
34d11ffa 3371
e01ad46d 3372 for (; j < end; j++) {
868f2f0b
KB
3373 struct blk_mq_hw_ctx *hctx = hctxs[j];
3374
3375 if (hctx) {
e155b0c2 3376 __blk_mq_free_map_and_rqs(set, j);
868f2f0b 3377 blk_mq_exit_hctx(q, set, hctx, j);
868f2f0b 3378 hctxs[j] = NULL;
868f2f0b
KB
3379 }
3380 }
fb350e0a 3381 mutex_unlock(&q->sysfs_lock);
868f2f0b
KB
3382}
3383
26a9750a
CH
3384int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
3385 struct request_queue *q)
868f2f0b 3386{
66841672
ML
3387 /* mark the queue as mq asap */
3388 q->mq_ops = set->ops;
3389
34dbad5d 3390 q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn,
720b8ccc
SB
3391 blk_mq_poll_stats_bkt,
3392 BLK_MQ_POLL_STATS_BKTS, q);
34dbad5d
OS
3393 if (!q->poll_cb)
3394 goto err_exit;
3395
1db4909e 3396 if (blk_mq_alloc_ctxs(q))
41de54c6 3397 goto err_poll;
868f2f0b 3398
737f98cf
ML
3399 /* init q->mq_kobj and sw queues' kobjects */
3400 blk_mq_sysfs_init(q);
3401
2f8f1336
ML
3402 INIT_LIST_HEAD(&q->unused_hctx_list);
3403 spin_lock_init(&q->unused_hctx_lock);
3404
868f2f0b
KB
3405 blk_mq_realloc_hw_ctxs(set, q);
3406 if (!q->nr_hw_queues)
3407 goto err_hctxs;
320ae51f 3408
287922eb 3409 INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
e56f698b 3410 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
320ae51f 3411
a8908939 3412 q->tag_set = set;
320ae51f 3413
94eddfbe 3414 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
cd19181b
ML
3415 if (set->nr_maps > HCTX_TYPE_POLL &&
3416 set->map[HCTX_TYPE_POLL].nr_queues)
6544d229 3417 blk_queue_flag_set(QUEUE_FLAG_POLL, q);
320ae51f 3418
2849450a 3419 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
6fca6a61
CH
3420 INIT_LIST_HEAD(&q->requeue_list);
3421 spin_lock_init(&q->requeue_lock);
3422
eba71768
JA
3423 q->nr_requests = set->queue_depth;
3424
64f1c21e
JA
3425 /*
3426 * Default to classic polling
3427 */
29ece8b4 3428 q->poll_nsec = BLK_MQ_POLL_CLASSIC;
64f1c21e 3429
24d2f903 3430 blk_mq_init_cpu_queues(q, set->nr_hw_queues);
0d2602ca 3431 blk_mq_add_queue_tag_set(set, q);
4b855ad3 3432 blk_mq_map_swqueue(q);
26a9750a 3433 return 0;
18741986 3434
320ae51f 3435err_hctxs:
868f2f0b 3436 kfree(q->queue_hw_ctx);
73d9c8d4 3437 q->nr_hw_queues = 0;
1db4909e 3438 blk_mq_sysfs_deinit(q);
41de54c6
JS
3439err_poll:
3440 blk_stat_free_callback(q->poll_cb);
3441 q->poll_cb = NULL;
c7de5726
ML
3442err_exit:
3443 q->mq_ops = NULL;
26a9750a 3444 return -ENOMEM;
320ae51f 3445}
b62c21b7 3446EXPORT_SYMBOL(blk_mq_init_allocated_queue);
320ae51f 3447
c7e2d94b
ML
3448/* tags can _not_ be used after returning from blk_mq_exit_queue */
3449void blk_mq_exit_queue(struct request_queue *q)
320ae51f 3450{
630ef623 3451 struct blk_mq_tag_set *set = q->tag_set;
320ae51f 3452
630ef623 3453 /* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */
624dbe47 3454 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
630ef623
BVA
3455 /* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */
3456 blk_mq_del_queue_tag_set(q);
320ae51f 3457}
320ae51f 3458
a5164405
JA
3459static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
3460{
3461 int i;
3462
079a2e3e
JG
3463 if (blk_mq_is_shared_tags(set->flags)) {
3464 set->shared_tags = blk_mq_alloc_map_and_rqs(set,
e155b0c2
JG
3465 BLK_MQ_NO_HCTX_IDX,
3466 set->queue_depth);
079a2e3e 3467 if (!set->shared_tags)
e155b0c2
JG
3468 return -ENOMEM;
3469 }
3470
8229cca8 3471 for (i = 0; i < set->nr_hw_queues; i++) {
63064be1 3472 if (!__blk_mq_alloc_map_and_rqs(set, i))
a5164405 3473 goto out_unwind;
8229cca8
XT
3474 cond_resched();
3475 }
a5164405
JA
3476
3477 return 0;
3478
3479out_unwind:
e155b0c2
JG
3480 while (--i >= 0)
3481 __blk_mq_free_map_and_rqs(set, i);
3482
079a2e3e
JG
3483 if (blk_mq_is_shared_tags(set->flags)) {
3484 blk_mq_free_map_and_rqs(set, set->shared_tags,
e155b0c2 3485 BLK_MQ_NO_HCTX_IDX);
645db34e 3486 }
a5164405 3487
a5164405
JA
3488 return -ENOMEM;
3489}
3490
3491/*
3492 * Allocate the request maps associated with this tag_set. Note that this
3493 * may reduce the depth asked for, if memory is tight. set->queue_depth
3494 * will be updated to reflect the allocated depth.
3495 */
63064be1 3496static int blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set *set)
a5164405
JA
3497{
3498 unsigned int depth;
3499 int err;
3500
3501 depth = set->queue_depth;
3502 do {
3503 err = __blk_mq_alloc_rq_maps(set);
3504 if (!err)
3505 break;
3506
3507 set->queue_depth >>= 1;
3508 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
3509 err = -ENOMEM;
3510 break;
3511 }
3512 } while (set->queue_depth);
3513
3514 if (!set->queue_depth || err) {
3515 pr_err("blk-mq: failed to allocate request map\n");
3516 return -ENOMEM;
3517 }
3518
3519 if (depth != set->queue_depth)
3520 pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
3521 depth, set->queue_depth);
3522
3523 return 0;
3524}
3525
ebe8bddb
OS
3526static int blk_mq_update_queue_map(struct blk_mq_tag_set *set)
3527{
6e66b493
BVA
3528 /*
3529 * blk_mq_map_queues() and multiple .map_queues() implementations
3530 * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the
3531 * number of hardware queues.
3532 */
3533 if (set->nr_maps == 1)
3534 set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues;
3535
59388702 3536 if (set->ops->map_queues && !is_kdump_kernel()) {
b3c661b1
JA
3537 int i;
3538
7d4901a9
ML
3539 /*
3540 * transport .map_queues is usually done in the following
3541 * way:
3542 *
3543 * for (queue = 0; queue < set->nr_hw_queues; queue++) {
3544 * mask = get_cpu_mask(queue)
3545 * for_each_cpu(cpu, mask)
b3c661b1 3546 * set->map[x].mq_map[cpu] = queue;
7d4901a9
ML
3547 * }
3548 *
3549 * When we need to remap, the table has to be cleared for
3550 * killing stale mapping since one CPU may not be mapped
3551 * to any hw queue.
3552 */
b3c661b1
JA
3553 for (i = 0; i < set->nr_maps; i++)
3554 blk_mq_clear_mq_map(&set->map[i]);
7d4901a9 3555
ebe8bddb 3556 return set->ops->map_queues(set);
b3c661b1
JA
3557 } else {
3558 BUG_ON(set->nr_maps > 1);
7d76f856 3559 return blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
b3c661b1 3560 }
ebe8bddb
OS
3561}
3562
f7e76dbc
BVA
3563static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set,
3564 int cur_nr_hw_queues, int new_nr_hw_queues)
3565{
3566 struct blk_mq_tags **new_tags;
3567
3568 if (cur_nr_hw_queues >= new_nr_hw_queues)
3569 return 0;
3570
3571 new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
3572 GFP_KERNEL, set->numa_node);
3573 if (!new_tags)
3574 return -ENOMEM;
3575
3576 if (set->tags)
3577 memcpy(new_tags, set->tags, cur_nr_hw_queues *
3578 sizeof(*set->tags));
3579 kfree(set->tags);
3580 set->tags = new_tags;
3581 set->nr_hw_queues = new_nr_hw_queues;
3582
3583 return 0;
3584}
3585
91cdf265
MI
3586static int blk_mq_alloc_tag_set_tags(struct blk_mq_tag_set *set,
3587 int new_nr_hw_queues)
3588{
3589 return blk_mq_realloc_tag_set_tags(set, 0, new_nr_hw_queues);
3590}
3591
a4391c64
JA
3592/*
3593 * Alloc a tag set to be associated with one or more request queues.
3594 * May fail with EINVAL for various error conditions. May adjust the
c018c84f 3595 * requested depth down, if it's too large. In that case, the set
a4391c64
JA
3596 * value will be stored in set->queue_depth.
3597 */
24d2f903
CH
3598int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
3599{
b3c661b1 3600 int i, ret;
da695ba2 3601
205fb5f5
BVA
3602 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
3603
24d2f903
CH
3604 if (!set->nr_hw_queues)
3605 return -EINVAL;
a4391c64 3606 if (!set->queue_depth)
24d2f903
CH
3607 return -EINVAL;
3608 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
3609 return -EINVAL;
3610
7d7e0f90 3611 if (!set->ops->queue_rq)
24d2f903
CH
3612 return -EINVAL;
3613
de148297
ML
3614 if (!set->ops->get_budget ^ !set->ops->put_budget)
3615 return -EINVAL;
3616
a4391c64
JA
3617 if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
3618 pr_info("blk-mq: reduced tag depth to %u\n",
3619 BLK_MQ_MAX_DEPTH);
3620 set->queue_depth = BLK_MQ_MAX_DEPTH;
3621 }
24d2f903 3622
b3c661b1
JA
3623 if (!set->nr_maps)
3624 set->nr_maps = 1;
3625 else if (set->nr_maps > HCTX_MAX_TYPES)
3626 return -EINVAL;
3627
6637fadf
SL
3628 /*
3629 * If a crashdump is active, then we are potentially in a very
3630 * memory constrained environment. Limit us to 1 queue and
3631 * 64 tags to prevent using too much memory.
3632 */
3633 if (is_kdump_kernel()) {
3634 set->nr_hw_queues = 1;
59388702 3635 set->nr_maps = 1;
6637fadf
SL
3636 set->queue_depth = min(64U, set->queue_depth);
3637 }
868f2f0b 3638 /*
392546ae
JA
3639 * There is no use for more h/w queues than cpus if we just have
3640 * a single map
868f2f0b 3641 */
392546ae 3642 if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
868f2f0b 3643 set->nr_hw_queues = nr_cpu_ids;
6637fadf 3644
91cdf265 3645 if (blk_mq_alloc_tag_set_tags(set, set->nr_hw_queues) < 0)
a5164405 3646 return -ENOMEM;
24d2f903 3647
da695ba2 3648 ret = -ENOMEM;
b3c661b1
JA
3649 for (i = 0; i < set->nr_maps; i++) {
3650 set->map[i].mq_map = kcalloc_node(nr_cpu_ids,
07b35eb5 3651 sizeof(set->map[i].mq_map[0]),
b3c661b1
JA
3652 GFP_KERNEL, set->numa_node);
3653 if (!set->map[i].mq_map)
3654 goto out_free_mq_map;
59388702 3655 set->map[i].nr_queues = is_kdump_kernel() ? 1 : set->nr_hw_queues;
b3c661b1 3656 }
bdd17e75 3657
ebe8bddb 3658 ret = blk_mq_update_queue_map(set);
da695ba2
CH
3659 if (ret)
3660 goto out_free_mq_map;
3661
63064be1 3662 ret = blk_mq_alloc_set_map_and_rqs(set);
da695ba2 3663 if (ret)
bdd17e75 3664 goto out_free_mq_map;
24d2f903 3665
0d2602ca
JA
3666 mutex_init(&set->tag_list_lock);
3667 INIT_LIST_HEAD(&set->tag_list);
3668
24d2f903 3669 return 0;
bdd17e75
CH
3670
3671out_free_mq_map:
b3c661b1
JA
3672 for (i = 0; i < set->nr_maps; i++) {
3673 kfree(set->map[i].mq_map);
3674 set->map[i].mq_map = NULL;
3675 }
5676e7b6
RE
3676 kfree(set->tags);
3677 set->tags = NULL;
da695ba2 3678 return ret;
24d2f903
CH
3679}
3680EXPORT_SYMBOL(blk_mq_alloc_tag_set);
3681
cdb14e0f
CH
3682/* allocate and initialize a tagset for a simple single-queue device */
3683int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
3684 const struct blk_mq_ops *ops, unsigned int queue_depth,
3685 unsigned int set_flags)
3686{
3687 memset(set, 0, sizeof(*set));
3688 set->ops = ops;
3689 set->nr_hw_queues = 1;
3690 set->nr_maps = 1;
3691 set->queue_depth = queue_depth;
3692 set->numa_node = NUMA_NO_NODE;
3693 set->flags = set_flags;
3694 return blk_mq_alloc_tag_set(set);
3695}
3696EXPORT_SYMBOL_GPL(blk_mq_alloc_sq_tag_set);
3697
24d2f903
CH
3698void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
3699{
b3c661b1 3700 int i, j;
24d2f903 3701
e155b0c2
JG
3702 for (i = 0; i < set->nr_hw_queues; i++)
3703 __blk_mq_free_map_and_rqs(set, i);
484b4061 3704
079a2e3e
JG
3705 if (blk_mq_is_shared_tags(set->flags)) {
3706 blk_mq_free_map_and_rqs(set, set->shared_tags,
e155b0c2
JG
3707 BLK_MQ_NO_HCTX_IDX);
3708 }
32bc15af 3709
b3c661b1
JA
3710 for (j = 0; j < set->nr_maps; j++) {
3711 kfree(set->map[j].mq_map);
3712 set->map[j].mq_map = NULL;
3713 }
bdd17e75 3714
981bd189 3715 kfree(set->tags);
5676e7b6 3716 set->tags = NULL;
24d2f903
CH
3717}
3718EXPORT_SYMBOL(blk_mq_free_tag_set);
3719
e3a2b3f9
JA
3720int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
3721{
3722 struct blk_mq_tag_set *set = q->tag_set;
3723 struct blk_mq_hw_ctx *hctx;
3724 int i, ret;
3725
bd166ef1 3726 if (!set)
e3a2b3f9
JA
3727 return -EINVAL;
3728
e5fa8140
AZ
3729 if (q->nr_requests == nr)
3730 return 0;
3731
70f36b60 3732 blk_mq_freeze_queue(q);
24f5a90f 3733 blk_mq_quiesce_queue(q);
70f36b60 3734
e3a2b3f9
JA
3735 ret = 0;
3736 queue_for_each_hw_ctx(q, hctx, i) {
e9137d4b
KB
3737 if (!hctx->tags)
3738 continue;
bd166ef1
JA
3739 /*
3740 * If we're using an MQ scheduler, just update the scheduler
3741 * queue depth. This is similar to what the old code would do.
3742 */
f6adcef5 3743 if (hctx->sched_tags) {
70f36b60 3744 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
f6adcef5 3745 nr, true);
f6adcef5
JG
3746 } else {
3747 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
3748 false);
70f36b60 3749 }
e3a2b3f9
JA
3750 if (ret)
3751 break;
77f1e0a5
JA
3752 if (q->elevator && q->elevator->type->ops.depth_updated)
3753 q->elevator->type->ops.depth_updated(hctx);
e3a2b3f9 3754 }
d97e594c 3755 if (!ret) {
e3a2b3f9 3756 q->nr_requests = nr;
079a2e3e 3757 if (blk_mq_is_shared_tags(set->flags)) {
8fa04464 3758 if (q->elevator)
079a2e3e 3759 blk_mq_tag_update_sched_shared_tags(q);
8fa04464 3760 else
079a2e3e 3761 blk_mq_tag_resize_shared_tags(set, nr);
8fa04464 3762 }
d97e594c 3763 }
e3a2b3f9 3764
24f5a90f 3765 blk_mq_unquiesce_queue(q);
70f36b60 3766 blk_mq_unfreeze_queue(q);
70f36b60 3767
e3a2b3f9
JA
3768 return ret;
3769}
3770
d48ece20
JW
3771/*
3772 * request_queue and elevator_type pair.
3773 * It is just used by __blk_mq_update_nr_hw_queues to cache
3774 * the elevator_type associated with a request_queue.
3775 */
3776struct blk_mq_qe_pair {
3777 struct list_head node;
3778 struct request_queue *q;
3779 struct elevator_type *type;
3780};
3781
3782/*
3783 * Cache the elevator_type in qe pair list and switch the
3784 * io scheduler to 'none'
3785 */
3786static bool blk_mq_elv_switch_none(struct list_head *head,
3787 struct request_queue *q)
3788{
3789 struct blk_mq_qe_pair *qe;
3790
3791 if (!q->elevator)
3792 return true;
3793
3794 qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY);
3795 if (!qe)
3796 return false;
3797
3798 INIT_LIST_HEAD(&qe->node);
3799 qe->q = q;
3800 qe->type = q->elevator->type;
3801 list_add(&qe->node, head);
3802
3803 mutex_lock(&q->sysfs_lock);
3804 /*
3805 * After elevator_switch_mq, the previous elevator_queue will be
3806 * released by elevator_release. The reference of the io scheduler
3807 * module get by elevator_get will also be put. So we need to get
3808 * a reference of the io scheduler module here to prevent it to be
3809 * removed.
3810 */
3811 __module_get(qe->type->elevator_owner);
3812 elevator_switch_mq(q, NULL);
3813 mutex_unlock(&q->sysfs_lock);
3814
3815 return true;
3816}
3817
3818static void blk_mq_elv_switch_back(struct list_head *head,
3819 struct request_queue *q)
3820{
3821 struct blk_mq_qe_pair *qe;
3822 struct elevator_type *t = NULL;
3823
3824 list_for_each_entry(qe, head, node)
3825 if (qe->q == q) {
3826 t = qe->type;
3827 break;
3828 }
3829
3830 if (!t)
3831 return;
3832
3833 list_del(&qe->node);
3834 kfree(qe);
3835
3836 mutex_lock(&q->sysfs_lock);
3837 elevator_switch_mq(q, t);
3838 mutex_unlock(&q->sysfs_lock);
3839}
3840
e4dc2b32
KB
3841static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
3842 int nr_hw_queues)
868f2f0b
KB
3843{
3844 struct request_queue *q;
d48ece20 3845 LIST_HEAD(head);
e01ad46d 3846 int prev_nr_hw_queues;
868f2f0b 3847
705cda97
BVA
3848 lockdep_assert_held(&set->tag_list_lock);
3849
392546ae 3850 if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids)
868f2f0b 3851 nr_hw_queues = nr_cpu_ids;
fe35ec58
WZ
3852 if (nr_hw_queues < 1)
3853 return;
3854 if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues)
868f2f0b
KB
3855 return;
3856
3857 list_for_each_entry(q, &set->tag_list, tag_set_list)
3858 blk_mq_freeze_queue(q);
d48ece20
JW
3859 /*
3860 * Switch IO scheduler to 'none', cleaning up the data associated
3861 * with the previous scheduler. We will switch back once we are done
3862 * updating the new sw to hw queue mappings.
3863 */
3864 list_for_each_entry(q, &set->tag_list, tag_set_list)
3865 if (!blk_mq_elv_switch_none(&head, q))
3866 goto switch_back;
868f2f0b 3867
477e19de
JW
3868 list_for_each_entry(q, &set->tag_list, tag_set_list) {
3869 blk_mq_debugfs_unregister_hctxs(q);
3870 blk_mq_sysfs_unregister(q);
3871 }
3872
a2584e43 3873 prev_nr_hw_queues = set->nr_hw_queues;
f7e76dbc
BVA
3874 if (blk_mq_realloc_tag_set_tags(set, set->nr_hw_queues, nr_hw_queues) <
3875 0)
3876 goto reregister;
3877
868f2f0b 3878 set->nr_hw_queues = nr_hw_queues;
e01ad46d 3879fallback:
aa880ad6 3880 blk_mq_update_queue_map(set);
868f2f0b
KB
3881 list_for_each_entry(q, &set->tag_list, tag_set_list) {
3882 blk_mq_realloc_hw_ctxs(set, q);
e01ad46d
JW
3883 if (q->nr_hw_queues != set->nr_hw_queues) {
3884 pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n",
3885 nr_hw_queues, prev_nr_hw_queues);
3886 set->nr_hw_queues = prev_nr_hw_queues;
7d76f856 3887 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
e01ad46d
JW
3888 goto fallback;
3889 }
477e19de
JW
3890 blk_mq_map_swqueue(q);
3891 }
3892
f7e76dbc 3893reregister:
477e19de
JW
3894 list_for_each_entry(q, &set->tag_list, tag_set_list) {
3895 blk_mq_sysfs_register(q);
3896 blk_mq_debugfs_register_hctxs(q);
868f2f0b
KB
3897 }
3898
d48ece20
JW
3899switch_back:
3900 list_for_each_entry(q, &set->tag_list, tag_set_list)
3901 blk_mq_elv_switch_back(&head, q);
3902
868f2f0b
KB
3903 list_for_each_entry(q, &set->tag_list, tag_set_list)
3904 blk_mq_unfreeze_queue(q);
3905}
e4dc2b32
KB
3906
3907void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
3908{
3909 mutex_lock(&set->tag_list_lock);
3910 __blk_mq_update_nr_hw_queues(set, nr_hw_queues);
3911 mutex_unlock(&set->tag_list_lock);
3912}
868f2f0b
KB
3913EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
3914
34dbad5d
OS
3915/* Enable polling stats and return whether they were already enabled. */
3916static bool blk_poll_stats_enable(struct request_queue *q)
3917{
3918 if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
7dfdbc73 3919 blk_queue_flag_test_and_set(QUEUE_FLAG_POLL_STATS, q))
34dbad5d
OS
3920 return true;
3921 blk_stat_add_callback(q, q->poll_cb);
3922 return false;
3923}
3924
3925static void blk_mq_poll_stats_start(struct request_queue *q)
3926{
3927 /*
3928 * We don't arm the callback if polling stats are not enabled or the
3929 * callback is already active.
3930 */
3931 if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
3932 blk_stat_is_active(q->poll_cb))
3933 return;
3934
3935 blk_stat_activate_msecs(q->poll_cb, 100);
3936}
3937
3938static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb)
3939{
3940 struct request_queue *q = cb->data;
720b8ccc 3941 int bucket;
34dbad5d 3942
720b8ccc
SB
3943 for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) {
3944 if (cb->stat[bucket].nr_samples)
3945 q->poll_stat[bucket] = cb->stat[bucket];
3946 }
34dbad5d
OS
3947}
3948
64f1c21e 3949static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
64f1c21e
JA
3950 struct request *rq)
3951{
64f1c21e 3952 unsigned long ret = 0;
720b8ccc 3953 int bucket;
64f1c21e
JA
3954
3955 /*
3956 * If stats collection isn't on, don't sleep but turn it on for
3957 * future users
3958 */
34dbad5d 3959 if (!blk_poll_stats_enable(q))
64f1c21e
JA
3960 return 0;
3961
64f1c21e
JA
3962 /*
3963 * As an optimistic guess, use half of the mean service time
3964 * for this type of request. We can (and should) make this smarter.
3965 * For instance, if the completion latencies are tight, we can
3966 * get closer than just half the mean. This is especially
3967 * important on devices where the completion latencies are longer
720b8ccc
SB
3968 * than ~10 usec. We do use the stats for the relevant IO size
3969 * if available which does lead to better estimates.
64f1c21e 3970 */
720b8ccc
SB
3971 bucket = blk_mq_poll_stats_bkt(rq);
3972 if (bucket < 0)
3973 return ret;
3974
3975 if (q->poll_stat[bucket].nr_samples)
3976 ret = (q->poll_stat[bucket].mean + 1) / 2;
64f1c21e
JA
3977
3978 return ret;
3979}
3980
c6699d6f 3981static bool blk_mq_poll_hybrid(struct request_queue *q, blk_qc_t qc)
06426adf 3982{
c6699d6f
CH
3983 struct blk_mq_hw_ctx *hctx = blk_qc_to_hctx(q, qc);
3984 struct request *rq = blk_qc_to_rq(hctx, qc);
06426adf
JA
3985 struct hrtimer_sleeper hs;
3986 enum hrtimer_mode mode;
64f1c21e 3987 unsigned int nsecs;
06426adf
JA
3988 ktime_t kt;
3989
c6699d6f
CH
3990 /*
3991 * If a request has completed on queue that uses an I/O scheduler, we
3992 * won't get back a request from blk_qc_to_rq.
3993 */
3994 if (!rq || (rq->rq_flags & RQF_MQ_POLL_SLEPT))
64f1c21e
JA
3995 return false;
3996
3997 /*
1052b8ac 3998 * If we get here, hybrid polling is enabled. Hence poll_nsec can be:
64f1c21e 3999 *
64f1c21e
JA
4000 * 0: use half of prev avg
4001 * >0: use this specific value
4002 */
1052b8ac 4003 if (q->poll_nsec > 0)
64f1c21e
JA
4004 nsecs = q->poll_nsec;
4005 else
cae740a0 4006 nsecs = blk_mq_poll_nsecs(q, rq);
64f1c21e
JA
4007
4008 if (!nsecs)
06426adf
JA
4009 return false;
4010
76a86f9d 4011 rq->rq_flags |= RQF_MQ_POLL_SLEPT;
06426adf
JA
4012
4013 /*
4014 * This will be replaced with the stats tracking code, using
4015 * 'avg_completion_time / 2' as the pre-sleep target.
4016 */
8b0e1953 4017 kt = nsecs;
06426adf
JA
4018
4019 mode = HRTIMER_MODE_REL;
dbc1625f 4020 hrtimer_init_sleeper_on_stack(&hs, CLOCK_MONOTONIC, mode);
06426adf
JA
4021 hrtimer_set_expires(&hs.timer, kt);
4022
06426adf 4023 do {
5a61c363 4024 if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE)
06426adf
JA
4025 break;
4026 set_current_state(TASK_UNINTERRUPTIBLE);
9dd8813e 4027 hrtimer_sleeper_start_expires(&hs, mode);
06426adf
JA
4028 if (hs.task)
4029 io_schedule();
4030 hrtimer_cancel(&hs.timer);
4031 mode = HRTIMER_MODE_ABS;
4032 } while (hs.task && !signal_pending(current));
4033
4034 __set_current_state(TASK_RUNNING);
4035 destroy_hrtimer_on_stack(&hs.timer);
c6699d6f
CH
4036
4037 /*
4038 * If we sleep, have the caller restart the poll loop to reset the
4039 * state. Like for the other success return cases, the caller is
4040 * responsible for checking if the IO completed. If the IO isn't
4041 * complete, we'll get called again and will go straight to the busy
4042 * poll loop.
4043 */
06426adf
JA
4044 return true;
4045}
4046
c6699d6f 4047static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie,
ef99b2d3 4048 unsigned int flags)
bbd7bb70 4049{
c6699d6f
CH
4050 struct blk_mq_hw_ctx *hctx = blk_qc_to_hctx(q, cookie);
4051 long state = get_current_state();
4052 int ret;
1052b8ac 4053
c6699d6f 4054 hctx->poll_considered++;
1052b8ac 4055
c6699d6f
CH
4056 do {
4057 hctx->poll_invoked++;
1052b8ac 4058
c6699d6f
CH
4059 ret = q->mq_ops->poll(hctx);
4060 if (ret > 0) {
4061 hctx->poll_success++;
4062 __set_current_state(TASK_RUNNING);
4063 return ret;
4064 }
4065
4066 if (signal_pending_state(state, current))
4067 __set_current_state(TASK_RUNNING);
4068 if (task_is_running(current))
4069 return 1;
4070
ef99b2d3 4071 if (ret < 0 || (flags & BLK_POLL_ONESHOT))
c6699d6f
CH
4072 break;
4073 cpu_relax();
4074 } while (!need_resched());
4075
4076 __set_current_state(TASK_RUNNING);
4077 return 0;
1052b8ac
JA
4078}
4079
3e08773c 4080int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, unsigned int flags)
1052b8ac 4081{
d729cf9a
CH
4082 if (!(flags & BLK_POLL_NOSLEEP) &&
4083 q->poll_nsec != BLK_MQ_POLL_CLASSIC) {
c6699d6f 4084 if (blk_mq_poll_hybrid(q, cookie))
85f4d4b6 4085 return 1;
c6699d6f 4086 }
ef99b2d3 4087 return blk_mq_poll_classic(q, cookie, flags);
bbd7bb70
JA
4088}
4089
9cf2bab6
JA
4090unsigned int blk_mq_rq_cpu(struct request *rq)
4091{
4092 return rq->mq_ctx->cpu;
4093}
4094EXPORT_SYMBOL(blk_mq_rq_cpu);
4095
320ae51f
JA
4096static int __init blk_mq_init(void)
4097{
c3077b5d
CH
4098 int i;
4099
4100 for_each_possible_cpu(i)
f9ab4918 4101 init_llist_head(&per_cpu(blk_cpu_done, i));
c3077b5d
CH
4102 open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
4103
4104 cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
4105 "block/softirq:dead", NULL,
4106 blk_softirq_cpu_dead);
9467f859
TG
4107 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
4108 blk_mq_hctx_notify_dead);
bf0beec0
ML
4109 cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
4110 blk_mq_hctx_notify_online,
4111 blk_mq_hctx_notify_offline);
320ae51f
JA
4112 return 0;
4113}
4114subsys_initcall(blk_mq_init);