Commit | Line | Data |
---|---|---|
3dcf60bc | 1 | // SPDX-License-Identifier: GPL-2.0 |
75bb4625 | 2 | /* |
88459642 OS |
3 | * Tag allocation using scalable bitmaps. Uses active queue tracking to support |
4 | * fairer distribution of tags between multiple submitters when a shared tag map | |
5 | * is used. | |
75bb4625 JA |
6 | * |
7 | * Copyright (C) 2013-2014 Jens Axboe | |
8 | */ | |
320ae51f JA |
9 | #include <linux/kernel.h> |
10 | #include <linux/module.h> | |
320ae51f | 11 | |
f9934a80 | 12 | #include <linux/delay.h> |
320ae51f JA |
13 | #include "blk.h" |
14 | #include "blk-mq.h" | |
d97e594c | 15 | #include "blk-mq-sched.h" |
320ae51f | 16 | |
180dccb0 LQ |
17 | /* |
18 | * Recalculate wakeup batch when tag is shared by hctx. | |
19 | */ | |
20 | static void blk_mq_update_wake_batch(struct blk_mq_tags *tags, | |
21 | unsigned int users) | |
22 | { | |
23 | if (!users) | |
24 | return; | |
25 | ||
26 | sbitmap_queue_recalculate_wake_batch(&tags->bitmap_tags, | |
27 | users); | |
28 | sbitmap_queue_recalculate_wake_batch(&tags->breserved_tags, | |
29 | users); | |
30 | } | |
31 | ||
0d2602ca JA |
32 | /* |
33 | * If a previously inactive queue goes active, bump the active user count. | |
d263ed99 JW |
34 | * We need to do this before try to allocate driver tag, then even if fail |
35 | * to get tag when first time, the other shared-tag users could reserve | |
36 | * budget for it. | |
0d2602ca | 37 | */ |
ee78ec10 | 38 | void __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) |
0d2602ca | 39 | { |
180dccb0 LQ |
40 | unsigned int users; |
41 | ||
079a2e3e | 42 | if (blk_mq_is_shared_tags(hctx->flags)) { |
f1b49fdc | 43 | struct request_queue *q = hctx->queue; |
f1b49fdc | 44 | |
ee78ec10 LS |
45 | if (test_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags)) |
46 | return; | |
47 | set_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags); | |
f1b49fdc | 48 | } else { |
ee78ec10 LS |
49 | if (test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) |
50 | return; | |
51 | set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state); | |
f1b49fdc | 52 | } |
0d2602ca | 53 | |
180dccb0 LQ |
54 | users = atomic_inc_return(&hctx->tags->active_queues); |
55 | ||
56 | blk_mq_update_wake_batch(hctx->tags, users); | |
0d2602ca JA |
57 | } |
58 | ||
59 | /* | |
aed3ea94 | 60 | * Wakeup all potentially sleeping on tags |
0d2602ca | 61 | */ |
aed3ea94 | 62 | void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve) |
0d2602ca | 63 | { |
ae0f1a73 | 64 | sbitmap_queue_wake_all(&tags->bitmap_tags); |
88459642 | 65 | if (include_reserve) |
ae0f1a73 | 66 | sbitmap_queue_wake_all(&tags->breserved_tags); |
0d2602ca JA |
67 | } |
68 | ||
e3a2b3f9 JA |
69 | /* |
70 | * If a previously busy queue goes inactive, potential waiters could now | |
71 | * be allowed to queue. Wake them up and check. | |
72 | */ | |
73 | void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) | |
74 | { | |
75 | struct blk_mq_tags *tags = hctx->tags; | |
180dccb0 | 76 | unsigned int users; |
e3a2b3f9 | 77 | |
079a2e3e | 78 | if (blk_mq_is_shared_tags(hctx->flags)) { |
e155b0c2 JG |
79 | struct request_queue *q = hctx->queue; |
80 | ||
f1b49fdc JG |
81 | if (!test_and_clear_bit(QUEUE_FLAG_HCTX_ACTIVE, |
82 | &q->queue_flags)) | |
83 | return; | |
f1b49fdc JG |
84 | } else { |
85 | if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) | |
86 | return; | |
f1b49fdc | 87 | } |
e3a2b3f9 | 88 | |
180dccb0 LQ |
89 | users = atomic_dec_return(&tags->active_queues); |
90 | ||
91 | blk_mq_update_wake_batch(tags, users); | |
079a2e3e | 92 | |
aed3ea94 | 93 | blk_mq_tag_wakeup_all(tags, false); |
e3a2b3f9 JA |
94 | } |
95 | ||
200e86b3 JA |
96 | static int __blk_mq_get_tag(struct blk_mq_alloc_data *data, |
97 | struct sbitmap_queue *bt) | |
4bb659b1 | 98 | { |
28500850 ML |
99 | if (!data->q->elevator && !(data->flags & BLK_MQ_REQ_RESERVED) && |
100 | !hctx_may_queue(data->hctx, bt)) | |
76647368 | 101 | return BLK_MQ_NO_TAG; |
42fdc5e4 | 102 | |
229a9287 | 103 | if (data->shallow_depth) |
3f607293 | 104 | return sbitmap_queue_get_shallow(bt, data->shallow_depth); |
229a9287 OS |
105 | else |
106 | return __sbitmap_queue_get(bt); | |
4bb659b1 JA |
107 | } |
108 | ||
349302da JA |
109 | unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags, |
110 | unsigned int *offset) | |
111 | { | |
112 | struct blk_mq_tags *tags = blk_mq_tags_from_data(data); | |
113 | struct sbitmap_queue *bt = &tags->bitmap_tags; | |
114 | unsigned long ret; | |
115 | ||
116 | if (data->shallow_depth ||data->flags & BLK_MQ_REQ_RESERVED || | |
117 | data->hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) | |
118 | return 0; | |
119 | ret = __sbitmap_queue_get_batch(bt, nr_tags, offset); | |
120 | *offset += tags->nr_reserved_tags; | |
121 | return ret; | |
122 | } | |
123 | ||
4941115b | 124 | unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) |
320ae51f | 125 | { |
4941115b JA |
126 | struct blk_mq_tags *tags = blk_mq_tags_from_data(data); |
127 | struct sbitmap_queue *bt; | |
88459642 | 128 | struct sbq_wait_state *ws; |
5d2ee712 | 129 | DEFINE_SBQ_WAIT(wait); |
4941115b | 130 | unsigned int tag_offset; |
320ae51f JA |
131 | int tag; |
132 | ||
4941115b JA |
133 | if (data->flags & BLK_MQ_REQ_RESERVED) { |
134 | if (unlikely(!tags->nr_reserved_tags)) { | |
135 | WARN_ON_ONCE(1); | |
419c3d5e | 136 | return BLK_MQ_NO_TAG; |
4941115b | 137 | } |
ae0f1a73 | 138 | bt = &tags->breserved_tags; |
4941115b JA |
139 | tag_offset = 0; |
140 | } else { | |
ae0f1a73 | 141 | bt = &tags->bitmap_tags; |
4941115b JA |
142 | tag_offset = tags->nr_reserved_tags; |
143 | } | |
144 | ||
200e86b3 | 145 | tag = __blk_mq_get_tag(data, bt); |
76647368 | 146 | if (tag != BLK_MQ_NO_TAG) |
4941115b | 147 | goto found_tag; |
4bb659b1 | 148 | |
6f3b0e8b | 149 | if (data->flags & BLK_MQ_REQ_NOWAIT) |
419c3d5e | 150 | return BLK_MQ_NO_TAG; |
4bb659b1 | 151 | |
4941115b | 152 | ws = bt_wait_ptr(bt, data->hctx); |
4bb659b1 | 153 | do { |
e6fc4649 ML |
154 | struct sbitmap_queue *bt_prev; |
155 | ||
b3223207 BVA |
156 | /* |
157 | * We're out of tags on this hardware queue, kick any | |
158 | * pending IO submits before going to sleep waiting for | |
8cecb07d | 159 | * some to complete. |
b3223207 | 160 | */ |
8cecb07d | 161 | blk_mq_run_hw_queue(data->hctx, false); |
b3223207 | 162 | |
080ff351 JA |
163 | /* |
164 | * Retry tag allocation after running the hardware queue, | |
165 | * as running the queue may also have found completions. | |
166 | */ | |
200e86b3 | 167 | tag = __blk_mq_get_tag(data, bt); |
76647368 | 168 | if (tag != BLK_MQ_NO_TAG) |
080ff351 JA |
169 | break; |
170 | ||
5d2ee712 | 171 | sbitmap_prepare_to_wait(bt, ws, &wait, TASK_UNINTERRUPTIBLE); |
4e5dff41 JA |
172 | |
173 | tag = __blk_mq_get_tag(data, bt); | |
76647368 | 174 | if (tag != BLK_MQ_NO_TAG) |
4e5dff41 JA |
175 | break; |
176 | ||
e6fc4649 | 177 | bt_prev = bt; |
4bb659b1 | 178 | io_schedule(); |
cb96a42c | 179 | |
5d2ee712 JA |
180 | sbitmap_finish_wait(bt, ws, &wait); |
181 | ||
cb96a42c | 182 | data->ctx = blk_mq_get_ctx(data->q); |
f9afca4d | 183 | data->hctx = blk_mq_map_queue(data->q, data->cmd_flags, |
8ccdf4a3 | 184 | data->ctx); |
4941115b JA |
185 | tags = blk_mq_tags_from_data(data); |
186 | if (data->flags & BLK_MQ_REQ_RESERVED) | |
ae0f1a73 | 187 | bt = &tags->breserved_tags; |
4941115b | 188 | else |
ae0f1a73 | 189 | bt = &tags->bitmap_tags; |
4941115b | 190 | |
e6fc4649 ML |
191 | /* |
192 | * If destination hw queue is changed, fake wake up on | |
193 | * previous queue for compensating the wake up miss, so | |
194 | * other allocations on previous queue won't be starved. | |
195 | */ | |
196 | if (bt != bt_prev) | |
4acb8341 | 197 | sbitmap_queue_wake_up(bt_prev, 1); |
e6fc4649 | 198 | |
4941115b | 199 | ws = bt_wait_ptr(bt, data->hctx); |
4bb659b1 JA |
200 | } while (1); |
201 | ||
5d2ee712 | 202 | sbitmap_finish_wait(bt, ws, &wait); |
320ae51f | 203 | |
4941115b | 204 | found_tag: |
bf0beec0 ML |
205 | /* |
206 | * Give up this allocation if the hctx is inactive. The caller will | |
207 | * retry on an active hctx. | |
208 | */ | |
209 | if (unlikely(test_bit(BLK_MQ_S_INACTIVE, &data->hctx->state))) { | |
210 | blk_mq_put_tag(tags, data->ctx, tag + tag_offset); | |
211 | return BLK_MQ_NO_TAG; | |
212 | } | |
4941115b | 213 | return tag + tag_offset; |
320ae51f JA |
214 | } |
215 | ||
cae740a0 JG |
216 | void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, |
217 | unsigned int tag) | |
320ae51f | 218 | { |
415b806d | 219 | if (!blk_mq_tag_is_reserved(tags, tag)) { |
4bb659b1 JA |
220 | const int real_tag = tag - tags->nr_reserved_tags; |
221 | ||
70114c39 | 222 | BUG_ON(real_tag >= tags->nr_tags); |
ae0f1a73 | 223 | sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu); |
70114c39 | 224 | } else { |
ae0f1a73 | 225 | sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu); |
70114c39 | 226 | } |
320ae51f JA |
227 | } |
228 | ||
f794f335 JA |
229 | void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags) |
230 | { | |
231 | sbitmap_queue_clear_batch(&tags->bitmap_tags, tags->nr_reserved_tags, | |
232 | tag_array, nr_tags); | |
233 | } | |
234 | ||
88459642 OS |
235 | struct bt_iter_data { |
236 | struct blk_mq_hw_ctx *hctx; | |
fea9f92f | 237 | struct request_queue *q; |
fc39f8d2 | 238 | busy_tag_iter_fn *fn; |
88459642 OS |
239 | void *data; |
240 | bool reserved; | |
241 | }; | |
242 | ||
2e315dc0 ML |
243 | static struct request *blk_mq_find_and_get_req(struct blk_mq_tags *tags, |
244 | unsigned int bitnr) | |
245 | { | |
bd63141d ML |
246 | struct request *rq; |
247 | unsigned long flags; | |
2e315dc0 | 248 | |
bd63141d ML |
249 | spin_lock_irqsave(&tags->lock, flags); |
250 | rq = tags->rqs[bitnr]; | |
0a467d0f | 251 | if (!rq || rq->tag != bitnr || !req_ref_inc_not_zero(rq)) |
bd63141d ML |
252 | rq = NULL; |
253 | spin_unlock_irqrestore(&tags->lock, flags); | |
2e315dc0 ML |
254 | return rq; |
255 | } | |
256 | ||
88459642 | 257 | static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) |
320ae51f | 258 | { |
88459642 OS |
259 | struct bt_iter_data *iter_data = data; |
260 | struct blk_mq_hw_ctx *hctx = iter_data->hctx; | |
fea9f92f JG |
261 | struct request_queue *q = iter_data->q; |
262 | struct blk_mq_tag_set *set = q->tag_set; | |
fea9f92f | 263 | struct blk_mq_tags *tags; |
81481eb4 | 264 | struct request *rq; |
2e315dc0 | 265 | bool ret = true; |
4bb659b1 | 266 | |
fea9f92f JG |
267 | if (blk_mq_is_shared_tags(set->flags)) |
268 | tags = set->shared_tags; | |
269 | else | |
270 | tags = hctx->tags; | |
271 | ||
4cf6e6c0 | 272 | if (!iter_data->reserved) |
88459642 | 273 | bitnr += tags->nr_reserved_tags; |
7f5562d5 JA |
274 | /* |
275 | * We can hit rq == NULL here, because the tagging functions | |
c7b1bf5c | 276 | * test and set the bit before assigning ->rqs[]. |
7f5562d5 | 277 | */ |
2e315dc0 ML |
278 | rq = blk_mq_find_and_get_req(tags, bitnr); |
279 | if (!rq) | |
280 | return true; | |
281 | ||
fea9f92f | 282 | if (rq->q == q && (!hctx || rq->mq_hctx == hctx)) |
2dd6532e | 283 | ret = iter_data->fn(rq, iter_data->data); |
2e315dc0 ML |
284 | blk_mq_put_rq_ref(rq); |
285 | return ret; | |
88459642 | 286 | } |
4bb659b1 | 287 | |
c7b1bf5c BVA |
288 | /** |
289 | * bt_for_each - iterate over the requests associated with a hardware queue | |
290 | * @hctx: Hardware queue to examine. | |
fea9f92f | 291 | * @q: Request queue to examine. |
c7b1bf5c BVA |
292 | * @bt: sbitmap to examine. This is either the breserved_tags member |
293 | * or the bitmap_tags member of struct blk_mq_tags. | |
294 | * @fn: Pointer to the function that will be called for each request | |
295 | * associated with @hctx that has been assigned a driver tag. | |
296 | * @fn will be called as follows: @fn(@hctx, rq, @data, @reserved) | |
ab11fe5a JA |
297 | * where rq is a pointer to a request. Return true to continue |
298 | * iterating tags, false to stop. | |
c7b1bf5c BVA |
299 | * @data: Will be passed as third argument to @fn. |
300 | * @reserved: Indicates whether @bt is the breserved_tags member or the | |
301 | * bitmap_tags member of struct blk_mq_tags. | |
302 | */ | |
fea9f92f JG |
303 | static void bt_for_each(struct blk_mq_hw_ctx *hctx, struct request_queue *q, |
304 | struct sbitmap_queue *bt, busy_tag_iter_fn *fn, | |
305 | void *data, bool reserved) | |
88459642 OS |
306 | { |
307 | struct bt_iter_data iter_data = { | |
308 | .hctx = hctx, | |
309 | .fn = fn, | |
310 | .data = data, | |
311 | .reserved = reserved, | |
fea9f92f | 312 | .q = q, |
88459642 OS |
313 | }; |
314 | ||
315 | sbitmap_for_each_set(&bt->sb, bt_iter, &iter_data); | |
320ae51f JA |
316 | } |
317 | ||
88459642 OS |
318 | struct bt_tags_iter_data { |
319 | struct blk_mq_tags *tags; | |
320 | busy_tag_iter_fn *fn; | |
321 | void *data; | |
602380d2 | 322 | unsigned int flags; |
88459642 OS |
323 | }; |
324 | ||
602380d2 ML |
325 | #define BT_TAG_ITER_RESERVED (1 << 0) |
326 | #define BT_TAG_ITER_STARTED (1 << 1) | |
22f614bc | 327 | #define BT_TAG_ITER_STATIC_RQS (1 << 2) |
602380d2 | 328 | |
88459642 | 329 | static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) |
f26cdc85 | 330 | { |
88459642 OS |
331 | struct bt_tags_iter_data *iter_data = data; |
332 | struct blk_mq_tags *tags = iter_data->tags; | |
f26cdc85 | 333 | struct request *rq; |
2e315dc0 ML |
334 | bool ret = true; |
335 | bool iter_static_rqs = !!(iter_data->flags & BT_TAG_ITER_STATIC_RQS); | |
f26cdc85 | 336 | |
4cf6e6c0 | 337 | if (!(iter_data->flags & BT_TAG_ITER_RESERVED)) |
88459642 | 338 | bitnr += tags->nr_reserved_tags; |
7f5562d5 JA |
339 | |
340 | /* | |
341 | * We can hit rq == NULL here, because the tagging functions | |
22f614bc | 342 | * test and set the bit before assigning ->rqs[]. |
7f5562d5 | 343 | */ |
2e315dc0 | 344 | if (iter_static_rqs) |
22f614bc ML |
345 | rq = tags->static_rqs[bitnr]; |
346 | else | |
2e315dc0 | 347 | rq = blk_mq_find_and_get_req(tags, bitnr); |
602380d2 ML |
348 | if (!rq) |
349 | return true; | |
2e315dc0 ML |
350 | |
351 | if (!(iter_data->flags & BT_TAG_ITER_STARTED) || | |
352 | blk_mq_request_started(rq)) | |
2dd6532e | 353 | ret = iter_data->fn(rq, iter_data->data); |
2e315dc0 ML |
354 | if (!iter_static_rqs) |
355 | blk_mq_put_rq_ref(rq); | |
356 | return ret; | |
88459642 OS |
357 | } |
358 | ||
c7b1bf5c BVA |
359 | /** |
360 | * bt_tags_for_each - iterate over the requests in a tag map | |
361 | * @tags: Tag map to iterate over. | |
362 | * @bt: sbitmap to examine. This is either the breserved_tags member | |
363 | * or the bitmap_tags member of struct blk_mq_tags. | |
364 | * @fn: Pointer to the function that will be called for each started | |
365 | * request. @fn will be called as follows: @fn(rq, @data, | |
ab11fe5a JA |
366 | * @reserved) where rq is a pointer to a request. Return true |
367 | * to continue iterating tags, false to stop. | |
c7b1bf5c | 368 | * @data: Will be passed as second argument to @fn. |
602380d2 | 369 | * @flags: BT_TAG_ITER_* |
c7b1bf5c | 370 | */ |
88459642 | 371 | static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt, |
602380d2 | 372 | busy_tag_iter_fn *fn, void *data, unsigned int flags) |
88459642 OS |
373 | { |
374 | struct bt_tags_iter_data iter_data = { | |
375 | .tags = tags, | |
376 | .fn = fn, | |
377 | .data = data, | |
602380d2 | 378 | .flags = flags, |
88459642 OS |
379 | }; |
380 | ||
381 | if (tags->rqs) | |
382 | sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data); | |
f26cdc85 KB |
383 | } |
384 | ||
602380d2 ML |
385 | static void __blk_mq_all_tag_iter(struct blk_mq_tags *tags, |
386 | busy_tag_iter_fn *fn, void *priv, unsigned int flags) | |
387 | { | |
388 | WARN_ON_ONCE(flags & BT_TAG_ITER_RESERVED); | |
389 | ||
390 | if (tags->nr_reserved_tags) | |
ae0f1a73 | 391 | bt_tags_for_each(tags, &tags->breserved_tags, fn, priv, |
602380d2 | 392 | flags | BT_TAG_ITER_RESERVED); |
ae0f1a73 | 393 | bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, flags); |
602380d2 ML |
394 | } |
395 | ||
c7b1bf5c | 396 | /** |
602380d2 | 397 | * blk_mq_all_tag_iter - iterate over all requests in a tag map |
c7b1bf5c | 398 | * @tags: Tag map to iterate over. |
602380d2 | 399 | * @fn: Pointer to the function that will be called for each |
c7b1bf5c BVA |
400 | * request. @fn will be called as follows: @fn(rq, @priv, |
401 | * reserved) where rq is a pointer to a request. 'reserved' | |
ab11fe5a JA |
402 | * indicates whether or not @rq is a reserved request. Return |
403 | * true to continue iterating tags, false to stop. | |
c7b1bf5c | 404 | * @priv: Will be passed as second argument to @fn. |
22f614bc ML |
405 | * |
406 | * Caller has to pass the tag map from which requests are allocated. | |
c7b1bf5c | 407 | */ |
602380d2 ML |
408 | void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn, |
409 | void *priv) | |
f26cdc85 | 410 | { |
a8a5e383 | 411 | __blk_mq_all_tag_iter(tags, fn, priv, BT_TAG_ITER_STATIC_RQS); |
f26cdc85 | 412 | } |
f26cdc85 | 413 | |
c7b1bf5c BVA |
414 | /** |
415 | * blk_mq_tagset_busy_iter - iterate over all started requests in a tag set | |
416 | * @tagset: Tag set to iterate over. | |
417 | * @fn: Pointer to the function that will be called for each started | |
418 | * request. @fn will be called as follows: @fn(rq, @priv, | |
419 | * reserved) where rq is a pointer to a request. 'reserved' | |
ab11fe5a JA |
420 | * indicates whether or not @rq is a reserved request. Return |
421 | * true to continue iterating tags, false to stop. | |
c7b1bf5c | 422 | * @priv: Will be passed as second argument to @fn. |
2e315dc0 ML |
423 | * |
424 | * We grab one request reference before calling @fn and release it after | |
425 | * @fn returns. | |
c7b1bf5c | 426 | */ |
e0489487 SG |
427 | void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, |
428 | busy_tag_iter_fn *fn, void *priv) | |
429 | { | |
0994c64e JG |
430 | unsigned int flags = tagset->flags; |
431 | int i, nr_tags; | |
e0489487 | 432 | |
0994c64e JG |
433 | nr_tags = blk_mq_is_shared_tags(flags) ? 1 : tagset->nr_hw_queues; |
434 | ||
435 | for (i = 0; i < nr_tags; i++) { | |
e0489487 | 436 | if (tagset->tags && tagset->tags[i]) |
602380d2 ML |
437 | __blk_mq_all_tag_iter(tagset->tags[i], fn, priv, |
438 | BT_TAG_ITER_STARTED); | |
e0489487 SG |
439 | } |
440 | } | |
441 | EXPORT_SYMBOL(blk_mq_tagset_busy_iter); | |
442 | ||
2dd6532e | 443 | static bool blk_mq_tagset_count_completed_rqs(struct request *rq, void *data) |
f9934a80 ML |
444 | { |
445 | unsigned *count = data; | |
446 | ||
447 | if (blk_mq_request_completed(rq)) | |
448 | (*count)++; | |
449 | return true; | |
450 | } | |
451 | ||
452 | /** | |
9cf1adc6 BC |
453 | * blk_mq_tagset_wait_completed_request - Wait until all scheduled request |
454 | * completions have finished. | |
f9934a80 ML |
455 | * @tagset: Tag set to drain completed request |
456 | * | |
457 | * Note: This function has to be run after all IO queues are shutdown | |
458 | */ | |
459 | void blk_mq_tagset_wait_completed_request(struct blk_mq_tag_set *tagset) | |
460 | { | |
461 | while (true) { | |
462 | unsigned count = 0; | |
463 | ||
464 | blk_mq_tagset_busy_iter(tagset, | |
465 | blk_mq_tagset_count_completed_rqs, &count); | |
466 | if (!count) | |
467 | break; | |
468 | msleep(5); | |
469 | } | |
470 | } | |
471 | EXPORT_SYMBOL(blk_mq_tagset_wait_completed_request); | |
472 | ||
c7b1bf5c BVA |
473 | /** |
474 | * blk_mq_queue_tag_busy_iter - iterate over all requests with a driver tag | |
475 | * @q: Request queue to examine. | |
476 | * @fn: Pointer to the function that will be called for each request | |
477 | * on @q. @fn will be called as follows: @fn(hctx, rq, @priv, | |
478 | * reserved) where rq is a pointer to a request and hctx points | |
479 | * to the hardware queue associated with the request. 'reserved' | |
480 | * indicates whether or not @rq is a reserved request. | |
481 | * @priv: Will be passed as third argument to @fn. | |
482 | * | |
483 | * Note: if @q->tag_set is shared with other request queues then @fn will be | |
484 | * called for all requests on all queues that share that tag set and not only | |
485 | * for requests associated with @q. | |
486 | */ | |
fc39f8d2 | 487 | void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn, |
81481eb4 | 488 | void *priv) |
320ae51f | 489 | { |
f5bbbbe4 | 490 | /* |
4e5cc99e | 491 | * __blk_mq_update_nr_hw_queues() updates nr_hw_queues and hctx_table |
c7b1bf5c | 492 | * while the queue is frozen. So we can use q_usage_counter to avoid |
76cffccd | 493 | * racing with it. |
f5bbbbe4 | 494 | */ |
530ca2c9 | 495 | if (!percpu_ref_tryget(&q->q_usage_counter)) |
f5bbbbe4 | 496 | return; |
0bf6cd5b | 497 | |
fea9f92f JG |
498 | if (blk_mq_is_shared_tags(q->tag_set->flags)) { |
499 | struct blk_mq_tags *tags = q->tag_set->shared_tags; | |
500 | struct sbitmap_queue *bresv = &tags->breserved_tags; | |
501 | struct sbitmap_queue *btags = &tags->bitmap_tags; | |
0bf6cd5b CH |
502 | |
503 | if (tags->nr_reserved_tags) | |
fea9f92f JG |
504 | bt_for_each(NULL, q, bresv, fn, priv, true); |
505 | bt_for_each(NULL, q, btags, fn, priv, false); | |
506 | } else { | |
507 | struct blk_mq_hw_ctx *hctx; | |
4f481208 | 508 | unsigned long i; |
fea9f92f JG |
509 | |
510 | queue_for_each_hw_ctx(q, hctx, i) { | |
511 | struct blk_mq_tags *tags = hctx->tags; | |
512 | struct sbitmap_queue *bresv = &tags->breserved_tags; | |
513 | struct sbitmap_queue *btags = &tags->bitmap_tags; | |
514 | ||
515 | /* | |
516 | * If no software queues are currently mapped to this | |
517 | * hardware queue, there's nothing to check | |
518 | */ | |
519 | if (!blk_mq_hw_queue_mapped(hctx)) | |
520 | continue; | |
521 | ||
522 | if (tags->nr_reserved_tags) | |
523 | bt_for_each(hctx, q, bresv, fn, priv, true); | |
524 | bt_for_each(hctx, q, btags, fn, priv, false); | |
525 | } | |
4bb659b1 | 526 | } |
530ca2c9 | 527 | blk_queue_exit(q); |
4bb659b1 JA |
528 | } |
529 | ||
f4a644db OS |
530 | static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth, |
531 | bool round_robin, int node) | |
4bb659b1 | 532 | { |
f4a644db OS |
533 | return sbitmap_queue_init_node(bt, depth, -1, round_robin, GFP_KERNEL, |
534 | node); | |
4bb659b1 JA |
535 | } |
536 | ||
56b68085 JG |
537 | int blk_mq_init_bitmaps(struct sbitmap_queue *bitmap_tags, |
538 | struct sbitmap_queue *breserved_tags, | |
539 | unsigned int queue_depth, unsigned int reserved, | |
540 | int node, int alloc_policy) | |
4bb659b1 | 541 | { |
56b68085 | 542 | unsigned int depth = queue_depth - reserved; |
f4a644db | 543 | bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR; |
4bb659b1 | 544 | |
56b68085 | 545 | if (bt_alloc(bitmap_tags, depth, round_robin, node)) |
4d063237 | 546 | return -ENOMEM; |
56b68085 | 547 | if (bt_alloc(breserved_tags, reserved, round_robin, node)) |
88459642 | 548 | goto free_bitmap_tags; |
4bb659b1 | 549 | |
56b68085 JG |
550 | return 0; |
551 | ||
552 | free_bitmap_tags: | |
553 | sbitmap_queue_free(bitmap_tags); | |
554 | return -ENOMEM; | |
555 | } | |
556 | ||
320ae51f | 557 | struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, |
24391c0d | 558 | unsigned int reserved_tags, |
e155b0c2 | 559 | int node, int alloc_policy) |
320ae51f | 560 | { |
320ae51f | 561 | struct blk_mq_tags *tags; |
320ae51f JA |
562 | |
563 | if (total_tags > BLK_MQ_TAG_MAX) { | |
564 | pr_err("blk-mq: tag depth too large\n"); | |
565 | return NULL; | |
566 | } | |
567 | ||
568 | tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node); | |
569 | if (!tags) | |
570 | return NULL; | |
571 | ||
320ae51f JA |
572 | tags->nr_tags = total_tags; |
573 | tags->nr_reserved_tags = reserved_tags; | |
bd63141d | 574 | spin_lock_init(&tags->lock); |
320ae51f | 575 | |
ae0f1a73 JG |
576 | if (blk_mq_init_bitmaps(&tags->bitmap_tags, &tags->breserved_tags, |
577 | total_tags, reserved_tags, node, | |
578 | alloc_policy) < 0) { | |
4d063237 HR |
579 | kfree(tags); |
580 | return NULL; | |
581 | } | |
582 | return tags; | |
320ae51f JA |
583 | } |
584 | ||
e155b0c2 | 585 | void blk_mq_free_tags(struct blk_mq_tags *tags) |
320ae51f | 586 | { |
ae0f1a73 JG |
587 | sbitmap_queue_free(&tags->bitmap_tags); |
588 | sbitmap_queue_free(&tags->breserved_tags); | |
320ae51f JA |
589 | kfree(tags); |
590 | } | |
591 | ||
70f36b60 JA |
592 | int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, |
593 | struct blk_mq_tags **tagsptr, unsigned int tdepth, | |
594 | bool can_grow) | |
e3a2b3f9 | 595 | { |
70f36b60 JA |
596 | struct blk_mq_tags *tags = *tagsptr; |
597 | ||
598 | if (tdepth <= tags->nr_reserved_tags) | |
e3a2b3f9 JA |
599 | return -EINVAL; |
600 | ||
601 | /* | |
70f36b60 JA |
602 | * If we are allowed to grow beyond the original size, allocate |
603 | * a new set of tags before freeing the old one. | |
e3a2b3f9 | 604 | */ |
70f36b60 JA |
605 | if (tdepth > tags->nr_tags) { |
606 | struct blk_mq_tag_set *set = hctx->queue->tag_set; | |
607 | struct blk_mq_tags *new; | |
70f36b60 JA |
608 | |
609 | if (!can_grow) | |
610 | return -EINVAL; | |
611 | ||
612 | /* | |
613 | * We need some sort of upper limit, set it high enough that | |
614 | * no valid use cases should require more. | |
615 | */ | |
d97e594c | 616 | if (tdepth > MAX_SCHED_RQ) |
70f36b60 JA |
617 | return -EINVAL; |
618 | ||
e155b0c2 JG |
619 | /* |
620 | * Only the sbitmap needs resizing since we allocated the max | |
621 | * initially. | |
622 | */ | |
079a2e3e | 623 | if (blk_mq_is_shared_tags(set->flags)) |
e155b0c2 JG |
624 | return 0; |
625 | ||
63064be1 | 626 | new = blk_mq_alloc_map_and_rqs(set, hctx->queue_num, tdepth); |
70f36b60 JA |
627 | if (!new) |
628 | return -ENOMEM; | |
70f36b60 | 629 | |
645db34e | 630 | blk_mq_free_map_and_rqs(set, *tagsptr, hctx->queue_num); |
70f36b60 JA |
631 | *tagsptr = new; |
632 | } else { | |
633 | /* | |
634 | * Don't need (or can't) update reserved tags here, they | |
635 | * remain static and should never need resizing. | |
636 | */ | |
ae0f1a73 | 637 | sbitmap_queue_resize(&tags->bitmap_tags, |
75d6e175 | 638 | tdepth - tags->nr_reserved_tags); |
70f36b60 | 639 | } |
88459642 | 640 | |
e3a2b3f9 JA |
641 | return 0; |
642 | } | |
643 | ||
079a2e3e | 644 | void blk_mq_tag_resize_shared_tags(struct blk_mq_tag_set *set, unsigned int size) |
32bc15af | 645 | { |
079a2e3e | 646 | struct blk_mq_tags *tags = set->shared_tags; |
e155b0c2 | 647 | |
ae0f1a73 | 648 | sbitmap_queue_resize(&tags->bitmap_tags, size - set->reserved_tags); |
32bc15af JG |
649 | } |
650 | ||
079a2e3e | 651 | void blk_mq_tag_update_sched_shared_tags(struct request_queue *q) |
a7e7388d | 652 | { |
079a2e3e | 653 | sbitmap_queue_resize(&q->sched_shared_tags->bitmap_tags, |
a7e7388d JG |
654 | q->nr_requests - q->tag_set->reserved_tags); |
655 | } | |
656 | ||
205fb5f5 BVA |
657 | /** |
658 | * blk_mq_unique_tag() - return a tag that is unique queue-wide | |
659 | * @rq: request for which to compute a unique tag | |
660 | * | |
661 | * The tag field in struct request is unique per hardware queue but not over | |
662 | * all hardware queues. Hence this function that returns a tag with the | |
663 | * hardware context index in the upper bits and the per hardware queue tag in | |
664 | * the lower bits. | |
665 | * | |
666 | * Note: When called for a request that is queued on a non-multiqueue request | |
667 | * queue, the hardware context index is set to zero. | |
668 | */ | |
669 | u32 blk_mq_unique_tag(struct request *rq) | |
670 | { | |
ea4f995e | 671 | return (rq->mq_hctx->queue_num << BLK_MQ_UNIQUE_TAG_BITS) | |
205fb5f5 BVA |
672 | (rq->tag & BLK_MQ_UNIQUE_TAG_MASK); |
673 | } | |
674 | EXPORT_SYMBOL(blk_mq_unique_tag); |