Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
320ae51f JA |
2 | #ifndef BLK_MQ_H |
3 | #define BLK_MQ_H | |
4 | ||
5 | #include <linux/blkdev.h> | |
88459642 | 6 | #include <linux/sbitmap.h> |
6a83e74d | 7 | #include <linux/srcu.h> |
320ae51f JA |
8 | |
9 | struct blk_mq_tags; | |
f70ced09 | 10 | struct blk_flush_queue; |
320ae51f | 11 | |
320ae51f JA |
12 | struct blk_mq_hw_ctx { |
13 | struct { | |
14 | spinlock_t lock; | |
15 | struct list_head dispatch; | |
8d354f13 | 16 | unsigned long state; /* BLK_MQ_S_* flags */ |
320ae51f JA |
17 | } ____cacheline_aligned_in_smp; |
18 | ||
9f993737 | 19 | struct delayed_work run_work; |
e4043dcf | 20 | cpumask_var_t cpumask; |
506e931f JA |
21 | int next_cpu; |
22 | int next_cpu_batch; | |
320ae51f JA |
23 | |
24 | unsigned long flags; /* BLK_MQ_F_* flags */ | |
25 | ||
bd166ef1 | 26 | void *sched_data; |
320ae51f | 27 | struct request_queue *queue; |
f70ced09 | 28 | struct blk_flush_queue *fq; |
320ae51f JA |
29 | |
30 | void *driver_data; | |
31 | ||
88459642 | 32 | struct sbitmap ctx_map; |
1429d7c9 | 33 | |
b347689f ML |
34 | struct blk_mq_ctx *dispatch_from; |
35 | ||
4bb659b1 | 36 | struct blk_mq_ctx **ctxs; |
8d354f13 | 37 | unsigned int nr_ctx; |
4bb659b1 | 38 | |
eb619fdb | 39 | wait_queue_entry_t dispatch_wait; |
8537b120 | 40 | atomic_t wait_index; |
320ae51f | 41 | |
320ae51f | 42 | struct blk_mq_tags *tags; |
bd166ef1 | 43 | struct blk_mq_tags *sched_tags; |
320ae51f JA |
44 | |
45 | unsigned long queued; | |
46 | unsigned long run; | |
8d354f13 | 47 | #define BLK_MQ_MAX_DISPATCH_ORDER 7 |
320ae51f JA |
48 | unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER]; |
49 | ||
320ae51f | 50 | unsigned int numa_node; |
17ded320 | 51 | unsigned int queue_num; |
320ae51f | 52 | |
0d2602ca JA |
53 | atomic_t nr_active; |
54 | ||
9467f859 | 55 | struct hlist_node cpuhp_dead; |
320ae51f | 56 | struct kobject kobj; |
05229bee | 57 | |
6e219353 | 58 | unsigned long poll_considered; |
05229bee JA |
59 | unsigned long poll_invoked; |
60 | unsigned long poll_success; | |
9c1051aa OS |
61 | |
62 | #ifdef CONFIG_BLK_DEBUG_FS | |
63 | struct dentry *debugfs_dir; | |
d332ce09 | 64 | struct dentry *sched_debugfs_dir; |
9c1051aa | 65 | #endif |
07319678 BVA |
66 | |
67 | /* Must be the last member - see also blk_mq_hw_ctx_size(). */ | |
68 | struct srcu_struct queue_rq_srcu[0]; | |
320ae51f JA |
69 | }; |
70 | ||
24d2f903 | 71 | struct blk_mq_tag_set { |
bdd17e75 | 72 | unsigned int *mq_map; |
f8a5b122 | 73 | const struct blk_mq_ops *ops; |
320ae51f | 74 | unsigned int nr_hw_queues; |
e3a2b3f9 | 75 | unsigned int queue_depth; /* max hw supported */ |
320ae51f JA |
76 | unsigned int reserved_tags; |
77 | unsigned int cmd_size; /* per-request extra data */ | |
78 | int numa_node; | |
79 | unsigned int timeout; | |
80 | unsigned int flags; /* BLK_MQ_F_* */ | |
24d2f903 CH |
81 | void *driver_data; |
82 | ||
83 | struct blk_mq_tags **tags; | |
0d2602ca JA |
84 | |
85 | struct mutex tag_list_lock; | |
86 | struct list_head tag_list; | |
320ae51f JA |
87 | }; |
88 | ||
74c45052 JA |
89 | struct blk_mq_queue_data { |
90 | struct request *rq; | |
74c45052 JA |
91 | bool last; |
92 | }; | |
93 | ||
fc17b653 CH |
94 | typedef blk_status_t (queue_rq_fn)(struct blk_mq_hw_ctx *, |
95 | const struct blk_mq_queue_data *); | |
88022d72 | 96 | typedef bool (get_budget_fn)(struct blk_mq_hw_ctx *); |
de148297 | 97 | typedef void (put_budget_fn)(struct blk_mq_hw_ctx *); |
0152fb6b | 98 | typedef enum blk_eh_timer_return (timeout_fn)(struct request *, bool); |
320ae51f JA |
99 | typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int); |
100 | typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int); | |
d6296d39 | 101 | typedef int (init_request_fn)(struct blk_mq_tag_set *set, struct request *, |
24d2f903 | 102 | unsigned int, unsigned int); |
d6296d39 | 103 | typedef void (exit_request_fn)(struct blk_mq_tag_set *set, struct request *, |
24d2f903 | 104 | unsigned int); |
320ae51f | 105 | |
81481eb4 CH |
106 | typedef void (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *, |
107 | bool); | |
f26cdc85 | 108 | typedef void (busy_tag_iter_fn)(struct request *, void *, bool); |
05229bee | 109 | typedef int (poll_fn)(struct blk_mq_hw_ctx *, unsigned int); |
da695ba2 | 110 | typedef int (map_queues_fn)(struct blk_mq_tag_set *set); |
05229bee | 111 | |
81481eb4 | 112 | |
320ae51f JA |
113 | struct blk_mq_ops { |
114 | /* | |
115 | * Queue request | |
116 | */ | |
117 | queue_rq_fn *queue_rq; | |
118 | ||
de148297 ML |
119 | /* |
120 | * Reserve budget before queue request, once .queue_rq is | |
121 | * run, it is driver's responsibility to release the | |
122 | * reserved budget. Also we have to handle failure case | |
123 | * of .get_budget for avoiding I/O deadlock. | |
124 | */ | |
125 | get_budget_fn *get_budget; | |
126 | put_budget_fn *put_budget; | |
127 | ||
320ae51f JA |
128 | /* |
129 | * Called on request timeout | |
130 | */ | |
0152fb6b | 131 | timeout_fn *timeout; |
320ae51f | 132 | |
05229bee JA |
133 | /* |
134 | * Called to poll for completion of a specific tag. | |
135 | */ | |
136 | poll_fn *poll; | |
137 | ||
30a91cb4 CH |
138 | softirq_done_fn *complete; |
139 | ||
320ae51f JA |
140 | /* |
141 | * Called when the block layer side of a hardware queue has been | |
142 | * set up, allowing the driver to allocate/init matching structures. | |
143 | * Ditto for exit/teardown. | |
144 | */ | |
145 | init_hctx_fn *init_hctx; | |
146 | exit_hctx_fn *exit_hctx; | |
e9b267d9 CH |
147 | |
148 | /* | |
149 | * Called for every command allocated by the block layer to allow | |
150 | * the driver to set up driver specific data. | |
f70ced09 ML |
151 | * |
152 | * Tag greater than or equal to queue_depth is for setting up | |
153 | * flush request. | |
154 | * | |
e9b267d9 CH |
155 | * Ditto for exit/teardown. |
156 | */ | |
157 | init_request_fn *init_request; | |
158 | exit_request_fn *exit_request; | |
d280bab3 BVA |
159 | /* Called from inside blk_get_request() */ |
160 | void (*initialize_rq_fn)(struct request *rq); | |
da695ba2 CH |
161 | |
162 | map_queues_fn *map_queues; | |
2836ee4b BVA |
163 | |
164 | #ifdef CONFIG_BLK_DEBUG_FS | |
165 | /* | |
166 | * Used by the debugfs implementation to show driver-specific | |
167 | * information about a request. | |
168 | */ | |
169 | void (*show_rq)(struct seq_file *m, struct request *rq); | |
170 | #endif | |
320ae51f JA |
171 | }; |
172 | ||
173 | enum { | |
320ae51f | 174 | BLK_MQ_F_SHOULD_MERGE = 1 << 0, |
8a58d1f1 JA |
175 | BLK_MQ_F_TAG_SHARED = 1 << 1, |
176 | BLK_MQ_F_SG_MERGE = 1 << 2, | |
1b792f2f | 177 | BLK_MQ_F_BLOCKING = 1 << 5, |
d3484991 | 178 | BLK_MQ_F_NO_SCHED = 1 << 6, |
24391c0d SL |
179 | BLK_MQ_F_ALLOC_POLICY_START_BIT = 8, |
180 | BLK_MQ_F_ALLOC_POLICY_BITS = 1, | |
320ae51f | 181 | |
5d12f905 | 182 | BLK_MQ_S_STOPPED = 0, |
0d2602ca | 183 | BLK_MQ_S_TAG_ACTIVE = 1, |
bd166ef1 | 184 | BLK_MQ_S_SCHED_RESTART = 2, |
eb619fdb | 185 | BLK_MQ_S_START_ON_RUN = 3, |
320ae51f | 186 | |
a4391c64 | 187 | BLK_MQ_MAX_DEPTH = 10240, |
506e931f JA |
188 | |
189 | BLK_MQ_CPU_WORK_BATCH = 8, | |
320ae51f | 190 | }; |
24391c0d SL |
191 | #define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \ |
192 | ((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \ | |
193 | ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) | |
194 | #define BLK_ALLOC_POLICY_TO_MQ_FLAG(policy) \ | |
195 | ((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \ | |
196 | << BLK_MQ_F_ALLOC_POLICY_START_BIT) | |
320ae51f | 197 | |
24d2f903 | 198 | struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); |
b62c21b7 MS |
199 | struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, |
200 | struct request_queue *q); | |
b21d5b30 MB |
201 | int blk_mq_register_dev(struct device *, struct request_queue *); |
202 | void blk_mq_unregister_dev(struct device *, struct request_queue *); | |
320ae51f | 203 | |
24d2f903 CH |
204 | int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set); |
205 | void blk_mq_free_tag_set(struct blk_mq_tag_set *set); | |
206 | ||
320ae51f JA |
207 | void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule); |
208 | ||
320ae51f JA |
209 | void blk_mq_free_request(struct request *rq); |
210 | bool blk_mq_can_queue(struct blk_mq_hw_ctx *); | |
6f3b0e8b CH |
211 | |
212 | enum { | |
9a95e4ef BVA |
213 | /* return when out of requests */ |
214 | BLK_MQ_REQ_NOWAIT = (__force blk_mq_req_flags_t)(1 << 0), | |
215 | /* allocate from reserved pool */ | |
216 | BLK_MQ_REQ_RESERVED = (__force blk_mq_req_flags_t)(1 << 1), | |
217 | /* allocate internal/sched tag */ | |
218 | BLK_MQ_REQ_INTERNAL = (__force blk_mq_req_flags_t)(1 << 2), | |
219 | /* set RQF_PREEMPT */ | |
220 | BLK_MQ_REQ_PREEMPT = (__force blk_mq_req_flags_t)(1 << 3), | |
6f3b0e8b CH |
221 | }; |
222 | ||
cd6ce148 | 223 | struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op, |
9a95e4ef | 224 | blk_mq_req_flags_t flags); |
cd6ce148 | 225 | struct request *blk_mq_alloc_request_hctx(struct request_queue *q, |
9a95e4ef BVA |
226 | unsigned int op, blk_mq_req_flags_t flags, |
227 | unsigned int hctx_idx); | |
0e62f51f | 228 | struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag); |
320ae51f | 229 | |
205fb5f5 BVA |
230 | enum { |
231 | BLK_MQ_UNIQUE_TAG_BITS = 16, | |
232 | BLK_MQ_UNIQUE_TAG_MASK = (1 << BLK_MQ_UNIQUE_TAG_BITS) - 1, | |
233 | }; | |
234 | ||
235 | u32 blk_mq_unique_tag(struct request *rq); | |
236 | ||
237 | static inline u16 blk_mq_unique_tag_to_hwq(u32 unique_tag) | |
238 | { | |
239 | return unique_tag >> BLK_MQ_UNIQUE_TAG_BITS; | |
240 | } | |
241 | ||
242 | static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag) | |
243 | { | |
244 | return unique_tag & BLK_MQ_UNIQUE_TAG_MASK; | |
245 | } | |
246 | ||
320ae51f | 247 | |
973c0191 | 248 | int blk_mq_request_started(struct request *rq); |
e2490073 | 249 | void blk_mq_start_request(struct request *rq); |
2a842aca CH |
250 | void blk_mq_end_request(struct request *rq, blk_status_t error); |
251 | void __blk_mq_end_request(struct request *rq, blk_status_t error); | |
320ae51f | 252 | |
2b053aca BVA |
253 | void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list); |
254 | void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, | |
255 | bool kick_requeue_list); | |
6fca6a61 | 256 | void blk_mq_kick_requeue_list(struct request_queue *q); |
2849450a | 257 | void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs); |
08e0029a | 258 | void blk_mq_complete_request(struct request *rq); |
30a91cb4 | 259 | |
fd001443 | 260 | bool blk_mq_queue_stopped(struct request_queue *q); |
320ae51f JA |
261 | void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx); |
262 | void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx); | |
280d45f6 | 263 | void blk_mq_stop_hw_queues(struct request_queue *q); |
2f268556 | 264 | void blk_mq_start_hw_queues(struct request_queue *q); |
ae911c5e | 265 | void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); |
1b4a3258 | 266 | void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async); |
97e01209 | 267 | void blk_mq_quiesce_queue(struct request_queue *q); |
e4e73913 | 268 | void blk_mq_unquiesce_queue(struct request_queue *q); |
7587a5ae | 269 | void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); |
79f720a7 | 270 | bool blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); |
b94ec296 | 271 | void blk_mq_run_hw_queues(struct request_queue *q, bool async); |
70f4db63 | 272 | void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); |
e0489487 SG |
273 | void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, |
274 | busy_tag_iter_fn *fn, void *priv); | |
c761d96b | 275 | void blk_mq_freeze_queue(struct request_queue *q); |
b4c6a028 | 276 | void blk_mq_unfreeze_queue(struct request_queue *q); |
1671d522 | 277 | void blk_freeze_queue_start(struct request_queue *q); |
6bae363e | 278 | void blk_mq_freeze_queue_wait(struct request_queue *q); |
f91328c4 KB |
279 | int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, |
280 | unsigned long timeout); | |
149e10f8 SG |
281 | int blk_mq_tagset_iter(struct blk_mq_tag_set *set, void *data, |
282 | int (reinit_request)(void *, struct request *)); | |
320ae51f | 283 | |
9e5a7e22 | 284 | int blk_mq_map_queues(struct blk_mq_tag_set *set); |
868f2f0b KB |
285 | void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues); |
286 | ||
852ec809 | 287 | void blk_mq_quiesce_queue_nowait(struct request_queue *q); |
4f084b41 | 288 | |
320ae51f JA |
289 | /* |
290 | * Driver command data is immediately after the request. So subtract request | |
2963e3f7 | 291 | * size to get back to the original request, add request size to get the PDU. |
320ae51f JA |
292 | */ |
293 | static inline struct request *blk_mq_rq_from_pdu(void *pdu) | |
294 | { | |
295 | return pdu - sizeof(struct request); | |
296 | } | |
297 | static inline void *blk_mq_rq_to_pdu(struct request *rq) | |
298 | { | |
2963e3f7 | 299 | return rq + 1; |
320ae51f JA |
300 | } |
301 | ||
320ae51f | 302 | #define queue_for_each_hw_ctx(q, hctx, i) \ |
0d0b7d42 JA |
303 | for ((i) = 0; (i) < (q)->nr_hw_queues && \ |
304 | ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++) | |
320ae51f | 305 | |
320ae51f | 306 | #define hctx_for_each_ctx(hctx, ctx, i) \ |
0d0b7d42 JA |
307 | for ((i) = 0; (i) < (hctx)->nr_ctx && \ |
308 | ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) | |
320ae51f | 309 | |
320ae51f | 310 | #endif |