Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
320ae51f JA |
2 | #ifndef BLK_MQ_H |
3 | #define BLK_MQ_H | |
4 | ||
5 | #include <linux/blkdev.h> | |
88459642 | 6 | #include <linux/sbitmap.h> |
6a83e74d | 7 | #include <linux/srcu.h> |
320ae51f JA |
8 | |
9 | struct blk_mq_tags; | |
f70ced09 | 10 | struct blk_flush_queue; |
320ae51f | 11 | |
fe644072 | 12 | /** |
d386732b AA |
13 | * struct blk_mq_hw_ctx - State for a hardware queue facing the hardware |
14 | * block device | |
fe644072 | 15 | */ |
320ae51f JA |
16 | struct blk_mq_hw_ctx { |
17 | struct { | |
d386732b | 18 | /** @lock: Protects the dispatch list. */ |
320ae51f | 19 | spinlock_t lock; |
d386732b AA |
20 | /** |
21 | * @dispatch: Used for requests that are ready to be | |
22 | * dispatched to the hardware but for some reason (e.g. lack of | |
23 | * resources) could not be sent to the hardware. As soon as the | |
24 | * driver can send new requests, requests at this list will | |
25 | * be sent first for a fairer dispatch. | |
26 | */ | |
320ae51f | 27 | struct list_head dispatch; |
d386732b AA |
28 | /** |
29 | * @state: BLK_MQ_S_* flags. Defines the state of the hw | |
30 | * queue (active, scheduled to restart, stopped). | |
31 | */ | |
32 | unsigned long state; | |
320ae51f JA |
33 | } ____cacheline_aligned_in_smp; |
34 | ||
d386732b AA |
35 | /** |
36 | * @run_work: Used for scheduling a hardware queue run at a later time. | |
37 | */ | |
9f993737 | 38 | struct delayed_work run_work; |
d386732b | 39 | /** @cpumask: Map of available CPUs where this hctx can run. */ |
e4043dcf | 40 | cpumask_var_t cpumask; |
d386732b AA |
41 | /** |
42 | * @next_cpu: Used by blk_mq_hctx_next_cpu() for round-robin CPU | |
43 | * selection from @cpumask. | |
44 | */ | |
506e931f | 45 | int next_cpu; |
d386732b AA |
46 | /** |
47 | * @next_cpu_batch: Counter of how many works left in the batch before | |
48 | * changing to the next CPU. | |
49 | */ | |
506e931f | 50 | int next_cpu_batch; |
320ae51f | 51 | |
d386732b AA |
52 | /** @flags: BLK_MQ_F_* flags. Defines the behaviour of the queue. */ |
53 | unsigned long flags; | |
320ae51f | 54 | |
d386732b AA |
55 | /** |
56 | * @sched_data: Pointer owned by the IO scheduler attached to a request | |
57 | * queue. It's up to the IO scheduler how to use this pointer. | |
58 | */ | |
bd166ef1 | 59 | void *sched_data; |
d386732b AA |
60 | /** |
61 | * @queue: Pointer to the request queue that owns this hardware context. | |
62 | */ | |
320ae51f | 63 | struct request_queue *queue; |
d386732b | 64 | /** @fq: Queue of requests that need to perform a flush operation. */ |
f70ced09 | 65 | struct blk_flush_queue *fq; |
320ae51f | 66 | |
d386732b AA |
67 | /** |
68 | * @driver_data: Pointer to data owned by the block driver that created | |
69 | * this hctx | |
70 | */ | |
320ae51f JA |
71 | void *driver_data; |
72 | ||
d386732b AA |
73 | /** |
74 | * @ctx_map: Bitmap for each software queue. If bit is on, there is a | |
75 | * pending request in that software queue. | |
76 | */ | |
88459642 | 77 | struct sbitmap ctx_map; |
1429d7c9 | 78 | |
d386732b AA |
79 | /** |
80 | * @dispatch_from: Software queue to be used when no scheduler was | |
81 | * selected. | |
82 | */ | |
b347689f | 83 | struct blk_mq_ctx *dispatch_from; |
d386732b AA |
84 | /** |
85 | * @dispatch_busy: Number used by blk_mq_update_dispatch_busy() to | |
86 | * decide if the hw_queue is busy using Exponential Weighted Moving | |
87 | * Average algorithm. | |
88 | */ | |
6e768717 | 89 | unsigned int dispatch_busy; |
b347689f | 90 | |
d386732b | 91 | /** @type: HCTX_TYPE_* flags. Type of hardware queue. */ |
f31967f0 | 92 | unsigned short type; |
d386732b | 93 | /** @nr_ctx: Number of software queues. */ |
f31967f0 | 94 | unsigned short nr_ctx; |
d386732b | 95 | /** @ctxs: Array of software queues. */ |
6e768717 | 96 | struct blk_mq_ctx **ctxs; |
4bb659b1 | 97 | |
d386732b | 98 | /** @dispatch_wait_lock: Lock for dispatch_wait queue. */ |
5815839b | 99 | spinlock_t dispatch_wait_lock; |
d386732b AA |
100 | /** |
101 | * @dispatch_wait: Waitqueue to put requests when there is no tag | |
102 | * available at the moment, to wait for another try in the future. | |
103 | */ | |
eb619fdb | 104 | wait_queue_entry_t dispatch_wait; |
d386732b AA |
105 | |
106 | /** | |
107 | * @wait_index: Index of next available dispatch_wait queue to insert | |
108 | * requests. | |
109 | */ | |
8537b120 | 110 | atomic_t wait_index; |
320ae51f | 111 | |
d386732b AA |
112 | /** |
113 | * @tags: Tags owned by the block driver. A tag at this set is only | |
114 | * assigned when a request is dispatched from a hardware queue. | |
115 | */ | |
320ae51f | 116 | struct blk_mq_tags *tags; |
d386732b AA |
117 | /** |
118 | * @sched_tags: Tags owned by I/O scheduler. If there is an I/O | |
119 | * scheduler associated with a request queue, a tag is assigned when | |
120 | * that request is allocated. Else, this member is not used. | |
121 | */ | |
bd166ef1 | 122 | struct blk_mq_tags *sched_tags; |
320ae51f | 123 | |
d386732b | 124 | /** @queued: Number of queued requests. */ |
320ae51f | 125 | unsigned long queued; |
d386732b | 126 | /** @run: Number of dispatched requests. */ |
320ae51f | 127 | unsigned long run; |
8d354f13 | 128 | #define BLK_MQ_MAX_DISPATCH_ORDER 7 |
d386732b | 129 | /** @dispatched: Number of dispatch requests by queue. */ |
320ae51f JA |
130 | unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER]; |
131 | ||
d386732b | 132 | /** @numa_node: NUMA node the storage adapter has been connected to. */ |
320ae51f | 133 | unsigned int numa_node; |
d386732b | 134 | /** @queue_num: Index of this hardware queue. */ |
17ded320 | 135 | unsigned int queue_num; |
320ae51f | 136 | |
d386732b AA |
137 | /** |
138 | * @nr_active: Number of active requests. Only used when a tag set is | |
139 | * shared across request queues. | |
140 | */ | |
0d2602ca JA |
141 | atomic_t nr_active; |
142 | ||
bf0beec0 ML |
143 | /** @cpuhp_online: List to store request if CPU is going to die */ |
144 | struct hlist_node cpuhp_online; | |
d386732b | 145 | /** @cpuhp_dead: List to store request if some CPU die. */ |
9467f859 | 146 | struct hlist_node cpuhp_dead; |
d386732b | 147 | /** @kobj: Kernel object for sysfs. */ |
320ae51f | 148 | struct kobject kobj; |
05229bee | 149 | |
d386732b | 150 | /** @poll_considered: Count times blk_poll() was called. */ |
6e219353 | 151 | unsigned long poll_considered; |
d386732b | 152 | /** @poll_invoked: Count how many requests blk_poll() polled. */ |
05229bee | 153 | unsigned long poll_invoked; |
d386732b | 154 | /** @poll_success: Count how many polled requests were completed. */ |
05229bee | 155 | unsigned long poll_success; |
9c1051aa OS |
156 | |
157 | #ifdef CONFIG_BLK_DEBUG_FS | |
d386732b AA |
158 | /** |
159 | * @debugfs_dir: debugfs directory for this hardware queue. Named | |
160 | * as cpu<cpu_number>. | |
161 | */ | |
9c1051aa | 162 | struct dentry *debugfs_dir; |
d386732b | 163 | /** @sched_debugfs_dir: debugfs directory for the scheduler. */ |
d332ce09 | 164 | struct dentry *sched_debugfs_dir; |
9c1051aa | 165 | #endif |
07319678 | 166 | |
2dd209f0 BVA |
167 | /** |
168 | * @hctx_list: if this hctx is not in use, this is an entry in | |
169 | * q->unused_hctx_list. | |
170 | */ | |
2f8f1336 ML |
171 | struct list_head hctx_list; |
172 | ||
d386732b AA |
173 | /** |
174 | * @srcu: Sleepable RCU. Use as lock when type of the hardware queue is | |
175 | * blocking (BLK_MQ_F_BLOCKING). Must be the last member - see also | |
176 | * blk_mq_hw_ctx_size(). | |
177 | */ | |
f36aaf8b | 178 | struct srcu_struct srcu[]; |
320ae51f JA |
179 | }; |
180 | ||
7a18312c | 181 | /** |
d386732b | 182 | * struct blk_mq_queue_map - Map software queues to hardware queues |
7a18312c BVA |
183 | * @mq_map: CPU ID to hardware queue index map. This is an array |
184 | * with nr_cpu_ids elements. Each element has a value in the range | |
185 | * [@queue_offset, @queue_offset + @nr_queues). | |
186 | * @nr_queues: Number of hardware queues to map CPU IDs onto. | |
187 | * @queue_offset: First hardware queue to map onto. Used by the PCIe NVMe | |
188 | * driver to map each hardware queue type (enum hctx_type) onto a distinct | |
189 | * set of hardware queues. | |
190 | */ | |
ed76e329 JA |
191 | struct blk_mq_queue_map { |
192 | unsigned int *mq_map; | |
193 | unsigned int nr_queues; | |
843477d4 | 194 | unsigned int queue_offset; |
ed76e329 JA |
195 | }; |
196 | ||
d386732b AA |
197 | /** |
198 | * enum hctx_type - Type of hardware queue | |
199 | * @HCTX_TYPE_DEFAULT: All I/O not otherwise accounted for. | |
200 | * @HCTX_TYPE_READ: Just for READ I/O. | |
201 | * @HCTX_TYPE_POLL: Polled I/O of any kind. | |
202 | * @HCTX_MAX_TYPES: Number of types of hctx. | |
203 | */ | |
e20ba6e1 | 204 | enum hctx_type { |
d386732b AA |
205 | HCTX_TYPE_DEFAULT, |
206 | HCTX_TYPE_READ, | |
207 | HCTX_TYPE_POLL, | |
e20ba6e1 CH |
208 | |
209 | HCTX_MAX_TYPES, | |
ed76e329 JA |
210 | }; |
211 | ||
7a18312c BVA |
212 | /** |
213 | * struct blk_mq_tag_set - tag set that can be shared between request queues | |
214 | * @map: One or more ctx -> hctx mappings. One map exists for each | |
215 | * hardware queue type (enum hctx_type) that the driver wishes | |
216 | * to support. There are no restrictions on maps being of the | |
217 | * same size, and it's perfectly legal to share maps between | |
218 | * types. | |
219 | * @nr_maps: Number of elements in the @map array. A number in the range | |
220 | * [1, HCTX_MAX_TYPES]. | |
221 | * @ops: Pointers to functions that implement block driver behavior. | |
222 | * @nr_hw_queues: Number of hardware queues supported by the block driver that | |
223 | * owns this data structure. | |
224 | * @queue_depth: Number of tags per hardware queue, reserved tags included. | |
225 | * @reserved_tags: Number of tags to set aside for BLK_MQ_REQ_RESERVED tag | |
226 | * allocations. | |
227 | * @cmd_size: Number of additional bytes to allocate per request. The block | |
228 | * driver owns these additional bytes. | |
229 | * @numa_node: NUMA node the storage adapter has been connected to. | |
230 | * @timeout: Request processing timeout in jiffies. | |
231 | * @flags: Zero or more BLK_MQ_F_* flags. | |
232 | * @driver_data: Pointer to data owned by the block driver that created this | |
233 | * tag set. | |
32bc15af JG |
234 | * @__bitmap_tags: A shared tags sbitmap, used over all hctx's |
235 | * @__breserved_tags: | |
236 | * A shared reserved tags sbitmap, used over all hctx's | |
7a18312c BVA |
237 | * @tags: Tag sets. One tag set per hardware queue. Has @nr_hw_queues |
238 | * elements. | |
239 | * @tag_list_lock: Serializes tag_list accesses. | |
240 | * @tag_list: List of the request queues that use this tag set. See also | |
241 | * request_queue.tag_set_list. | |
242 | */ | |
24d2f903 | 243 | struct blk_mq_tag_set { |
ed76e329 | 244 | struct blk_mq_queue_map map[HCTX_MAX_TYPES]; |
7a18312c | 245 | unsigned int nr_maps; |
f8a5b122 | 246 | const struct blk_mq_ops *ops; |
7a18312c BVA |
247 | unsigned int nr_hw_queues; |
248 | unsigned int queue_depth; | |
320ae51f | 249 | unsigned int reserved_tags; |
7a18312c | 250 | unsigned int cmd_size; |
320ae51f JA |
251 | int numa_node; |
252 | unsigned int timeout; | |
7a18312c | 253 | unsigned int flags; |
24d2f903 CH |
254 | void *driver_data; |
255 | ||
32bc15af JG |
256 | struct sbitmap_queue __bitmap_tags; |
257 | struct sbitmap_queue __breserved_tags; | |
24d2f903 | 258 | struct blk_mq_tags **tags; |
0d2602ca JA |
259 | |
260 | struct mutex tag_list_lock; | |
261 | struct list_head tag_list; | |
320ae51f JA |
262 | }; |
263 | ||
d386732b AA |
264 | /** |
265 | * struct blk_mq_queue_data - Data about a request inserted in a queue | |
266 | * | |
267 | * @rq: Request pointer. | |
268 | * @last: If it is the last request in the queue. | |
269 | */ | |
74c45052 JA |
270 | struct blk_mq_queue_data { |
271 | struct request *rq; | |
74c45052 JA |
272 | bool last; |
273 | }; | |
274 | ||
7baa8572 | 275 | typedef bool (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *, |
81481eb4 | 276 | bool); |
7baa8572 | 277 | typedef bool (busy_tag_iter_fn)(struct request *, void *, bool); |
05229bee | 278 | |
d386732b AA |
279 | /** |
280 | * struct blk_mq_ops - Callback functions that implements block driver | |
281 | * behaviour. | |
282 | */ | |
320ae51f | 283 | struct blk_mq_ops { |
d386732b AA |
284 | /** |
285 | * @queue_rq: Queue a new request from block IO. | |
320ae51f | 286 | */ |
0516c2f6 DW |
287 | blk_status_t (*queue_rq)(struct blk_mq_hw_ctx *, |
288 | const struct blk_mq_queue_data *); | |
320ae51f | 289 | |
d386732b AA |
290 | /** |
291 | * @commit_rqs: If a driver uses bd->last to judge when to submit | |
292 | * requests to hardware, it must define this function. In case of errors | |
293 | * that make us stop issuing further requests, this hook serves the | |
d666ba98 JA |
294 | * purpose of kicking the hardware (which the last request otherwise |
295 | * would have done). | |
296 | */ | |
0516c2f6 | 297 | void (*commit_rqs)(struct blk_mq_hw_ctx *); |
d666ba98 | 298 | |
d386732b AA |
299 | /** |
300 | * @get_budget: Reserve budget before queue request, once .queue_rq is | |
de148297 ML |
301 | * run, it is driver's responsibility to release the |
302 | * reserved budget. Also we have to handle failure case | |
303 | * of .get_budget for avoiding I/O deadlock. | |
304 | */ | |
0516c2f6 DW |
305 | bool (*get_budget)(struct request_queue *); |
306 | ||
d386732b AA |
307 | /** |
308 | * @put_budget: Release the reserved budget. | |
309 | */ | |
0516c2f6 | 310 | void (*put_budget)(struct request_queue *); |
de148297 | 311 | |
d386732b AA |
312 | /** |
313 | * @timeout: Called on request timeout. | |
320ae51f | 314 | */ |
0516c2f6 | 315 | enum blk_eh_timer_return (*timeout)(struct request *, bool); |
320ae51f | 316 | |
d386732b AA |
317 | /** |
318 | * @poll: Called to poll for completion of a specific tag. | |
05229bee | 319 | */ |
0516c2f6 | 320 | int (*poll)(struct blk_mq_hw_ctx *); |
05229bee | 321 | |
d386732b AA |
322 | /** |
323 | * @complete: Mark the request as complete. | |
324 | */ | |
0516c2f6 | 325 | void (*complete)(struct request *); |
30a91cb4 | 326 | |
d386732b AA |
327 | /** |
328 | * @init_hctx: Called when the block layer side of a hardware queue has | |
329 | * been set up, allowing the driver to allocate/init matching | |
330 | * structures. | |
320ae51f | 331 | */ |
0516c2f6 | 332 | int (*init_hctx)(struct blk_mq_hw_ctx *, void *, unsigned int); |
d386732b AA |
333 | /** |
334 | * @exit_hctx: Ditto for exit/teardown. | |
335 | */ | |
0516c2f6 | 336 | void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int); |
e9b267d9 | 337 | |
d386732b AA |
338 | /** |
339 | * @init_request: Called for every command allocated by the block layer | |
340 | * to allow the driver to set up driver specific data. | |
f70ced09 ML |
341 | * |
342 | * Tag greater than or equal to queue_depth is for setting up | |
343 | * flush request. | |
e9b267d9 | 344 | */ |
0516c2f6 DW |
345 | int (*init_request)(struct blk_mq_tag_set *set, struct request *, |
346 | unsigned int, unsigned int); | |
d386732b AA |
347 | /** |
348 | * @exit_request: Ditto for exit/teardown. | |
349 | */ | |
0516c2f6 DW |
350 | void (*exit_request)(struct blk_mq_tag_set *set, struct request *, |
351 | unsigned int); | |
d386732b AA |
352 | |
353 | /** | |
354 | * @initialize_rq_fn: Called from inside blk_get_request(). | |
355 | */ | |
d280bab3 | 356 | void (*initialize_rq_fn)(struct request *rq); |
da695ba2 | 357 | |
d386732b AA |
358 | /** |
359 | * @cleanup_rq: Called before freeing one request which isn't completed | |
360 | * yet, and usually for freeing the driver private data. | |
226b4fc7 | 361 | */ |
0516c2f6 | 362 | void (*cleanup_rq)(struct request *); |
226b4fc7 | 363 | |
d386732b AA |
364 | /** |
365 | * @busy: If set, returns whether or not this queue currently is busy. | |
9ba20527 | 366 | */ |
0516c2f6 | 367 | bool (*busy)(struct request_queue *); |
9ba20527 | 368 | |
d386732b AA |
369 | /** |
370 | * @map_queues: This allows drivers specify their own queue mapping by | |
371 | * overriding the setup-time function that builds the mq_map. | |
372 | */ | |
0516c2f6 | 373 | int (*map_queues)(struct blk_mq_tag_set *set); |
2836ee4b BVA |
374 | |
375 | #ifdef CONFIG_BLK_DEBUG_FS | |
d386732b AA |
376 | /** |
377 | * @show_rq: Used by the debugfs implementation to show driver-specific | |
2836ee4b BVA |
378 | * information about a request. |
379 | */ | |
380 | void (*show_rq)(struct seq_file *m, struct request *rq); | |
381 | #endif | |
320ae51f JA |
382 | }; |
383 | ||
384 | enum { | |
320ae51f | 385 | BLK_MQ_F_SHOULD_MERGE = 1 << 0, |
51db1c37 | 386 | BLK_MQ_F_TAG_QUEUE_SHARED = 1 << 1, |
bf0beec0 ML |
387 | /* |
388 | * Set when this device requires underlying blk-mq device for | |
389 | * completing IO: | |
390 | */ | |
391 | BLK_MQ_F_STACKING = 1 << 2, | |
32bc15af | 392 | BLK_MQ_F_TAG_HCTX_SHARED = 1 << 3, |
1b792f2f | 393 | BLK_MQ_F_BLOCKING = 1 << 5, |
d3484991 | 394 | BLK_MQ_F_NO_SCHED = 1 << 6, |
24391c0d SL |
395 | BLK_MQ_F_ALLOC_POLICY_START_BIT = 8, |
396 | BLK_MQ_F_ALLOC_POLICY_BITS = 1, | |
320ae51f | 397 | |
5d12f905 | 398 | BLK_MQ_S_STOPPED = 0, |
0d2602ca | 399 | BLK_MQ_S_TAG_ACTIVE = 1, |
bd166ef1 | 400 | BLK_MQ_S_SCHED_RESTART = 2, |
320ae51f | 401 | |
bf0beec0 ML |
402 | /* hw queue is inactive after all its CPUs become offline */ |
403 | BLK_MQ_S_INACTIVE = 3, | |
404 | ||
a4391c64 | 405 | BLK_MQ_MAX_DEPTH = 10240, |
506e931f JA |
406 | |
407 | BLK_MQ_CPU_WORK_BATCH = 8, | |
320ae51f | 408 | }; |
24391c0d SL |
409 | #define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \ |
410 | ((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \ | |
411 | ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) | |
412 | #define BLK_ALLOC_POLICY_TO_MQ_FLAG(policy) \ | |
413 | ((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \ | |
414 | << BLK_MQ_F_ALLOC_POLICY_START_BIT) | |
320ae51f | 415 | |
24d2f903 | 416 | struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); |
2f227bb9 CH |
417 | struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set, |
418 | void *queuedata); | |
b62c21b7 | 419 | struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, |
737eb78e DLM |
420 | struct request_queue *q, |
421 | bool elevator_init); | |
9316a9ed JA |
422 | struct request_queue *blk_mq_init_sq_queue(struct blk_mq_tag_set *set, |
423 | const struct blk_mq_ops *ops, | |
424 | unsigned int queue_depth, | |
425 | unsigned int set_flags); | |
b21d5b30 | 426 | void blk_mq_unregister_dev(struct device *, struct request_queue *); |
320ae51f | 427 | |
24d2f903 CH |
428 | int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set); |
429 | void blk_mq_free_tag_set(struct blk_mq_tag_set *set); | |
430 | ||
320ae51f JA |
431 | void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule); |
432 | ||
320ae51f | 433 | void blk_mq_free_request(struct request *rq); |
6f3b0e8b | 434 | |
3c94d83c | 435 | bool blk_mq_queue_inflight(struct request_queue *q); |
ae879912 | 436 | |
6f3b0e8b | 437 | enum { |
9a95e4ef BVA |
438 | /* return when out of requests */ |
439 | BLK_MQ_REQ_NOWAIT = (__force blk_mq_req_flags_t)(1 << 0), | |
440 | /* allocate from reserved pool */ | |
441 | BLK_MQ_REQ_RESERVED = (__force blk_mq_req_flags_t)(1 << 1), | |
9a95e4ef BVA |
442 | /* set RQF_PREEMPT */ |
443 | BLK_MQ_REQ_PREEMPT = (__force blk_mq_req_flags_t)(1 << 3), | |
6f3b0e8b CH |
444 | }; |
445 | ||
cd6ce148 | 446 | struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op, |
9a95e4ef | 447 | blk_mq_req_flags_t flags); |
cd6ce148 | 448 | struct request *blk_mq_alloc_request_hctx(struct request_queue *q, |
9a95e4ef BVA |
449 | unsigned int op, blk_mq_req_flags_t flags, |
450 | unsigned int hctx_idx); | |
0e62f51f | 451 | struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag); |
320ae51f | 452 | |
205fb5f5 BVA |
453 | enum { |
454 | BLK_MQ_UNIQUE_TAG_BITS = 16, | |
455 | BLK_MQ_UNIQUE_TAG_MASK = (1 << BLK_MQ_UNIQUE_TAG_BITS) - 1, | |
456 | }; | |
457 | ||
458 | u32 blk_mq_unique_tag(struct request *rq); | |
459 | ||
460 | static inline u16 blk_mq_unique_tag_to_hwq(u32 unique_tag) | |
461 | { | |
462 | return unique_tag >> BLK_MQ_UNIQUE_TAG_BITS; | |
463 | } | |
464 | ||
465 | static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag) | |
466 | { | |
467 | return unique_tag & BLK_MQ_UNIQUE_TAG_MASK; | |
468 | } | |
469 | ||
27a46989 PB |
470 | /** |
471 | * blk_mq_rq_state() - read the current MQ_RQ_* state of a request | |
472 | * @rq: target request. | |
473 | */ | |
474 | static inline enum mq_rq_state blk_mq_rq_state(struct request *rq) | |
475 | { | |
476 | return READ_ONCE(rq->state); | |
477 | } | |
478 | ||
479 | static inline int blk_mq_request_started(struct request *rq) | |
480 | { | |
481 | return blk_mq_rq_state(rq) != MQ_RQ_IDLE; | |
482 | } | |
483 | ||
484 | static inline int blk_mq_request_completed(struct request *rq) | |
485 | { | |
486 | return blk_mq_rq_state(rq) == MQ_RQ_COMPLETE; | |
487 | } | |
320ae51f | 488 | |
e2490073 | 489 | void blk_mq_start_request(struct request *rq); |
2a842aca CH |
490 | void blk_mq_end_request(struct request *rq, blk_status_t error); |
491 | void __blk_mq_end_request(struct request *rq, blk_status_t error); | |
320ae51f | 492 | |
2b053aca | 493 | void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list); |
6fca6a61 | 494 | void blk_mq_kick_requeue_list(struct request_queue *q); |
2849450a | 495 | void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs); |
15f73f5b | 496 | void blk_mq_complete_request(struct request *rq); |
40d09b53 | 497 | bool blk_mq_complete_request_remote(struct request *rq); |
fd001443 | 498 | bool blk_mq_queue_stopped(struct request_queue *q); |
320ae51f JA |
499 | void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx); |
500 | void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx); | |
280d45f6 | 501 | void blk_mq_stop_hw_queues(struct request_queue *q); |
2f268556 | 502 | void blk_mq_start_hw_queues(struct request_queue *q); |
ae911c5e | 503 | void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); |
1b4a3258 | 504 | void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async); |
97e01209 | 505 | void blk_mq_quiesce_queue(struct request_queue *q); |
e4e73913 | 506 | void blk_mq_unquiesce_queue(struct request_queue *q); |
7587a5ae | 507 | void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); |
626fb735 | 508 | void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); |
b94ec296 | 509 | void blk_mq_run_hw_queues(struct request_queue *q, bool async); |
b9151e7b | 510 | void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs); |
e0489487 SG |
511 | void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, |
512 | busy_tag_iter_fn *fn, void *priv); | |
f9934a80 | 513 | void blk_mq_tagset_wait_completed_request(struct blk_mq_tag_set *tagset); |
c761d96b | 514 | void blk_mq_freeze_queue(struct request_queue *q); |
b4c6a028 | 515 | void blk_mq_unfreeze_queue(struct request_queue *q); |
1671d522 | 516 | void blk_freeze_queue_start(struct request_queue *q); |
6bae363e | 517 | void blk_mq_freeze_queue_wait(struct request_queue *q); |
f91328c4 KB |
518 | int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, |
519 | unsigned long timeout); | |
320ae51f | 520 | |
ed76e329 | 521 | int blk_mq_map_queues(struct blk_mq_queue_map *qmap); |
868f2f0b KB |
522 | void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues); |
523 | ||
852ec809 | 524 | void blk_mq_quiesce_queue_nowait(struct request_queue *q); |
4f084b41 | 525 | |
9cf2bab6 JA |
526 | unsigned int blk_mq_rq_cpu(struct request *rq); |
527 | ||
15f73f5b CH |
528 | bool __blk_should_fake_timeout(struct request_queue *q); |
529 | static inline bool blk_should_fake_timeout(struct request_queue *q) | |
530 | { | |
531 | if (IS_ENABLED(CONFIG_FAIL_IO_TIMEOUT) && | |
532 | test_bit(QUEUE_FLAG_FAIL_IO, &q->queue_flags)) | |
533 | return __blk_should_fake_timeout(q); | |
534 | return false; | |
535 | } | |
536 | ||
d386732b AA |
537 | /** |
538 | * blk_mq_rq_from_pdu - cast a PDU to a request | |
539 | * @pdu: the PDU (Protocol Data Unit) to be casted | |
540 | * | |
541 | * Return: request | |
542 | * | |
320ae51f | 543 | * Driver command data is immediately after the request. So subtract request |
d386732b | 544 | * size to get back to the original request. |
320ae51f JA |
545 | */ |
546 | static inline struct request *blk_mq_rq_from_pdu(void *pdu) | |
547 | { | |
548 | return pdu - sizeof(struct request); | |
549 | } | |
d386732b AA |
550 | |
551 | /** | |
552 | * blk_mq_rq_to_pdu - cast a request to a PDU | |
553 | * @rq: the request to be casted | |
554 | * | |
555 | * Return: pointer to the PDU | |
556 | * | |
557 | * Driver command data is immediately after the request. So add request to get | |
558 | * the PDU. | |
559 | */ | |
320ae51f JA |
560 | static inline void *blk_mq_rq_to_pdu(struct request *rq) |
561 | { | |
2963e3f7 | 562 | return rq + 1; |
320ae51f JA |
563 | } |
564 | ||
320ae51f | 565 | #define queue_for_each_hw_ctx(q, hctx, i) \ |
0d0b7d42 JA |
566 | for ((i) = 0; (i) < (q)->nr_hw_queues && \ |
567 | ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++) | |
320ae51f | 568 | |
320ae51f | 569 | #define hctx_for_each_ctx(hctx, ctx, i) \ |
0d0b7d42 JA |
570 | for ((i) = 0; (i) < (hctx)->nr_ctx && \ |
571 | ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) | |
320ae51f | 572 | |
7b7ab780 SG |
573 | static inline blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, |
574 | struct request *rq) | |
575 | { | |
576 | if (rq->tag != -1) | |
577 | return rq->tag | (hctx->queue_num << BLK_QC_T_SHIFT); | |
578 | ||
579 | return rq->internal_tag | (hctx->queue_num << BLK_QC_T_SHIFT) | | |
580 | BLK_QC_T_INTERNAL; | |
581 | } | |
582 | ||
226b4fc7 ML |
583 | static inline void blk_mq_cleanup_rq(struct request *rq) |
584 | { | |
585 | if (rq->q->mq_ops->cleanup_rq) | |
586 | rq->q->mq_ops->cleanup_rq(rq); | |
587 | } | |
588 | ||
c62b37d9 | 589 | blk_qc_t blk_mq_submit_bio(struct bio *bio); |
8cf7961d | 590 | |
320ae51f | 591 | #endif |