blk-mq: fix two misuses on RQF_USE_SCHED
[linux-block.git] / block / blk.h
CommitLineData
b2441318 1/* SPDX-License-Identifier: GPL-2.0 */
8324aa91
JA
2#ifndef BLK_INTERNAL_H
3#define BLK_INTERNAL_H
4
a892c8d5 5#include <linux/blk-crypto.h>
9bb33f24 6#include <linux/memblock.h> /* for max_pfn/max_low_pfn */
c39ae60d 7#include <xen/xen.h>
a892c8d5 8#include "blk-crypto-internal.h"
a73f730d 9
2e9bc346
CH
10struct elevator_type;
11
0d2602ca
JA
12/* Max future timer expiry for timeouts */
13#define BLK_MAX_TIMEOUT (5 * HZ)
14
18fbda91 15extern struct dentry *blk_debugfs_root;
18fbda91 16
7c94e1c1 17struct blk_flush_queue {
7c94e1c1
ML
18 unsigned int flush_pending_idx:1;
19 unsigned int flush_running_idx:1;
8d699663 20 blk_status_t rq_status;
7c94e1c1
ML
21 unsigned long flush_pending_since;
22 struct list_head flush_queue[2];
23 struct list_head flush_data_in_flight;
24 struct request *flush_rq;
0048b483 25
7c94e1c1
ML
26 spinlock_t mq_flush_lock;
27};
28
a9ed27a7 29bool is_flush_rq(struct request *req);
8d699663 30
754a1572
GJ
31struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size,
32 gfp_t flags);
f70ced09 33void blk_free_flush_queue(struct blk_flush_queue *q);
f3552655 34
3ef28e83 35void blk_freeze_queue(struct request_queue *q);
aec89dc5 36void __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic);
8e141f9e 37void blk_queue_start_drain(struct request_queue *q);
c98cb5bb 38int __bio_queue_enter(struct request_queue *q, struct bio *bio);
3f98c753 39void submit_bio_noacct_nocheck(struct bio *bio);
c98cb5bb
JA
40
41static inline bool blk_try_enter_queue(struct request_queue *q, bool pm)
42{
43 rcu_read_lock();
44 if (!percpu_ref_tryget_live_rcu(&q->q_usage_counter))
45 goto fail;
46
47 /*
48 * The code that increments the pm_only counter must ensure that the
49 * counter is globally visible before the queue is unfrozen.
50 */
51 if (blk_queue_pm_only(q) &&
52 (!pm || queue_rpm_status(q) == RPM_SUSPENDED))
53 goto fail_put;
54
55 rcu_read_unlock();
56 return true;
57
58fail_put:
59 blk_queue_exit(q);
60fail:
61 rcu_read_unlock();
62 return false;
63}
64
65static inline int bio_queue_enter(struct bio *bio)
66{
67 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
68
69 if (blk_try_enter_queue(q, false))
70 return 0;
71 return __bio_queue_enter(q, bio);
72}
3ef28e83 73
dc0b8a57 74#define BIO_INLINE_VECS 4
7a800a20
CH
75struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
76 gfp_t gfp_mask);
77void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs);
eec716a1 78
3dccdae5
CH
79static inline bool biovec_phys_mergeable(struct request_queue *q,
80 struct bio_vec *vec1, struct bio_vec *vec2)
6a9f5f24 81{
3dccdae5 82 unsigned long mask = queue_segment_boundary(q);
6e768461
CH
83 phys_addr_t addr1 = page_to_phys(vec1->bv_page) + vec1->bv_offset;
84 phys_addr_t addr2 = page_to_phys(vec2->bv_page) + vec2->bv_offset;
3dccdae5 85
f630a5d0
AP
86 /*
87 * Merging adjacent physical pages may not work correctly under KMSAN
88 * if their metadata pages aren't adjacent. Just disable merging.
89 */
90 if (IS_ENABLED(CONFIG_KMSAN))
91 return false;
92
3dccdae5 93 if (addr1 + vec1->bv_len != addr2)
6a9f5f24 94 return false;
0383ad43 95 if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page))
6a9f5f24 96 return false;
3dccdae5
CH
97 if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask))
98 return false;
6a9f5f24
CH
99 return true;
100}
101
aa261f20 102static inline bool __bvec_gap_to_prev(const struct queue_limits *lim,
27ca1d4e
CH
103 struct bio_vec *bprv, unsigned int offset)
104{
c55ddd90
CH
105 return (offset & lim->virt_boundary_mask) ||
106 ((bprv->bv_offset + bprv->bv_len) & lim->virt_boundary_mask);
27ca1d4e
CH
107}
108
109/*
110 * Check if adding a bio_vec after bprv with offset would create a gap in
111 * the SG list. Most drivers don't care about this, but some do.
112 */
aa261f20 113static inline bool bvec_gap_to_prev(const struct queue_limits *lim,
27ca1d4e
CH
114 struct bio_vec *bprv, unsigned int offset)
115{
c55ddd90 116 if (!lim->virt_boundary_mask)
27ca1d4e 117 return false;
c55ddd90 118 return __bvec_gap_to_prev(lim, bprv, offset);
27ca1d4e
CH
119}
120
badf7f64
CH
121static inline bool rq_mergeable(struct request *rq)
122{
123 if (blk_rq_is_passthrough(rq))
124 return false;
125
126 if (req_op(rq) == REQ_OP_FLUSH)
127 return false;
128
129 if (req_op(rq) == REQ_OP_WRITE_ZEROES)
130 return false;
131
132 if (req_op(rq) == REQ_OP_ZONE_APPEND)
133 return false;
134
135 if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
136 return false;
137 if (rq->rq_flags & RQF_NOMERGE_FLAGS)
138 return false;
139
140 return true;
141}
142
143/*
144 * There are two different ways to handle DISCARD merges:
145 * 1) If max_discard_segments > 1, the driver treats every bio as a range and
146 * send the bios to controller together. The ranges don't need to be
147 * contiguous.
148 * 2) Otherwise, the request will be normal read/write requests. The ranges
149 * need to be contiguous.
150 */
151static inline bool blk_discard_mergable(struct request *req)
152{
153 if (req_op(req) == REQ_OP_DISCARD &&
154 queue_max_discard_segments(req->q) > 1)
155 return true;
156 return false;
157}
158
49d24398
US
159static inline unsigned int blk_rq_get_max_segments(struct request *rq)
160{
161 if (req_op(rq) == REQ_OP_DISCARD)
162 return queue_max_discard_segments(rq->q);
163 return queue_max_segments(rq->q);
164}
165
2a9336c4 166static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
77e7ffd7 167 enum req_op op)
2a9336c4
CH
168{
169 if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE))
170 return min(q->limits.max_discard_sectors,
171 UINT_MAX >> SECTOR_SHIFT);
172
173 if (unlikely(op == REQ_OP_WRITE_ZEROES))
174 return q->limits.max_write_zeroes_sectors;
175
176 return q->limits.max_sectors;
177}
178
5a48fc14
DW
179#ifdef CONFIG_BLK_DEV_INTEGRITY
180void blk_flush_integrity(void);
7c20f116 181bool __bio_integrity_endio(struct bio *);
ece841ab 182void bio_integrity_free(struct bio *bio);
7c20f116
CH
183static inline bool bio_integrity_endio(struct bio *bio)
184{
185 if (bio_integrity(bio))
186 return __bio_integrity_endio(bio);
187 return true;
188}
43b729bf 189
92cf2fd1
CH
190bool blk_integrity_merge_rq(struct request_queue *, struct request *,
191 struct request *);
d59da419
CH
192bool blk_integrity_merge_bio(struct request_queue *, struct request *,
193 struct bio *);
92cf2fd1 194
43b729bf
CH
195static inline bool integrity_req_gap_back_merge(struct request *req,
196 struct bio *next)
197{
198 struct bio_integrity_payload *bip = bio_integrity(req->bio);
199 struct bio_integrity_payload *bip_next = bio_integrity(next);
200
c55ddd90
CH
201 return bvec_gap_to_prev(&req->q->limits,
202 &bip->bip_vec[bip->bip_vcnt - 1],
43b729bf
CH
203 bip_next->bip_vec[0].bv_offset);
204}
205
206static inline bool integrity_req_gap_front_merge(struct request *req,
207 struct bio *bio)
208{
209 struct bio_integrity_payload *bip = bio_integrity(bio);
210 struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
211
c55ddd90
CH
212 return bvec_gap_to_prev(&req->q->limits,
213 &bip->bip_vec[bip->bip_vcnt - 1],
43b729bf
CH
214 bip_next->bip_vec[0].bv_offset);
215}
581e2600 216
ff53cd52 217extern const struct attribute_group blk_integrity_attr_group;
43b729bf 218#else /* CONFIG_BLK_DEV_INTEGRITY */
92cf2fd1
CH
219static inline bool blk_integrity_merge_rq(struct request_queue *rq,
220 struct request *r1, struct request *r2)
221{
222 return true;
223}
d59da419
CH
224static inline bool blk_integrity_merge_bio(struct request_queue *rq,
225 struct request *r, struct bio *b)
226{
227 return true;
228}
43b729bf
CH
229static inline bool integrity_req_gap_back_merge(struct request *req,
230 struct bio *next)
231{
232 return false;
233}
234static inline bool integrity_req_gap_front_merge(struct request *req,
235 struct bio *bio)
236{
237 return false;
238}
239
5a48fc14
DW
240static inline void blk_flush_integrity(void)
241{
242}
7c20f116
CH
243static inline bool bio_integrity_endio(struct bio *bio)
244{
245 return true;
246}
ece841ab
JT
247static inline void bio_integrity_free(struct bio *bio)
248{
249}
43b729bf 250#endif /* CONFIG_BLK_DEV_INTEGRITY */
8324aa91 251
0d2602ca 252unsigned long blk_rq_timeout(unsigned long timeout);
87ee7b11 253void blk_add_timer(struct request *req);
0d7a29a2 254const char *blk_status_to_str(blk_status_t status);
320ae51f 255
320ae51f 256bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
0c5bcc92 257 unsigned int nr_segs);
bdc6a287
BW
258bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
259 struct bio *bio, unsigned int nr_segs);
320ae51f 260
ba0ffdd8
JA
261/*
262 * Plug flush limits
263 */
264#define BLK_MAX_REQUEST_COUNT 32
265#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
266
158dbda0
TH
267/*
268 * Internal elevator interface
269 */
e8064021 270#define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED)
158dbda0 271
360f2648 272bool blk_insert_flush(struct request *rq);
dd831006 273
8237c01f 274int elevator_switch(struct request_queue *q, struct elevator_type *new_e);
64b36075 275void elevator_disable(struct request_queue *q);
0c6cb3a2 276void elevator_exit(struct request_queue *q);
cecf5d87 277int elv_register_queue(struct request_queue *q, bool uevent);
83d016ac
BVA
278void elv_unregister_queue(struct request_queue *q);
279
3ad5cee5
CH
280ssize_t part_size_show(struct device *dev, struct device_attribute *attr,
281 char *buf);
282ssize_t part_stat_show(struct device *dev, struct device_attribute *attr,
283 char *buf);
284ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr,
285 char *buf);
286ssize_t part_fail_show(struct device *dev, struct device_attribute *attr,
287 char *buf);
288ssize_t part_fail_store(struct device *dev, struct device_attribute *attr,
289 const char *buf, size_t count);
581d4e28
JA
290ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
291ssize_t part_timeout_store(struct device *, struct device_attribute *,
292 const char *, size_t);
581d4e28 293
c55ddd90 294static inline bool bio_may_exceed_limits(struct bio *bio,
aa261f20 295 const struct queue_limits *lim)
abd45c15
JA
296{
297 switch (bio_op(bio)) {
298 case REQ_OP_DISCARD:
299 case REQ_OP_SECURE_ERASE:
300 case REQ_OP_WRITE_ZEROES:
abd45c15
JA
301 return true; /* non-trivial splitting decisions */
302 default:
303 break;
304 }
305
306 /*
307 * All drivers must accept single-segments bios that are <= PAGE_SIZE.
308 * This is a quick and dirty check that relies on the fact that
309 * bi_io_vec[0] is always valid if a bio has data. The check might
310 * lead to occasional false negatives when bios are cloned, but compared
311 * to the performance impact of cloned bios themselves the loop below
312 * doesn't matter anyway.
313 */
c55ddd90 314 return lim->chunk_sectors || bio->bi_vcnt != 1 ||
abd45c15
JA
315 bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > PAGE_SIZE;
316}
317
aa261f20
BVA
318struct bio *__bio_split_to_limits(struct bio *bio,
319 const struct queue_limits *lim,
320 unsigned int *nr_segs);
14ccb66b
CH
321int ll_back_merge_fn(struct request *req, struct bio *bio,
322 unsigned int nr_segs);
fd2ef39c 323bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
5e84ea3a 324 struct request *next);
e9cd19c0 325unsigned int blk_recalc_rq_segments(struct request *rq);
80a761fd 326void blk_rq_set_mixed_merge(struct request *rq);
050c8ea8 327bool blk_rq_merge_ok(struct request *rq, struct bio *bio);
34fe7c05 328enum elv_merge blk_try_merge(struct request *rq, struct bio *bio);
d6d48196 329
b3228254 330void blk_set_default_limits(struct queue_limits *lim);
ff88972c
AB
331int blk_dev_init(void);
332
c2553b58
JA
333/*
334 * Contribute to IO statistics IFF:
335 *
336 * a) it's attached to a gendisk, and
48d9b0d4 337 * b) the queue had IO stats enabled when this request was started
c2553b58 338 */
599d067d 339static inline bool blk_do_io_stat(struct request *rq)
fb8ec18c 340{
41fa7222 341 return (rq->rq_flags & RQF_IO_STAT) && !blk_rq_is_passthrough(rq);
be6bfe36
PB
342}
343
450b7879 344void update_io_ticks(struct block_device *part, unsigned long now, bool end);
fb8ec18c 345
6cf7677f
CH
346static inline void req_set_nomerge(struct request_queue *q, struct request *req)
347{
348 req->cmd_flags |= REQ_NOMERGE;
349 if (req == q->last_merge)
350 q->last_merge = NULL;
351}
352
f2dbd76a
TH
353/*
354 * Internal io_context interface
355 */
87dd1d63 356struct io_cq *ioc_find_get_icq(struct request_queue *q);
eca5892a 357struct io_cq *ioc_lookup_icq(struct request_queue *q);
5ef16305 358#ifdef CONFIG_BLK_ICQ
7e5a8794 359void ioc_clear_queue(struct request_queue *q);
5ef16305
CH
360#else
361static inline void ioc_clear_queue(struct request_queue *q)
362{
363}
364#endif /* CONFIG_BLK_ICQ */
f2dbd76a 365
297e3d85
SL
366#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
367extern ssize_t blk_throtl_sample_time_show(struct request_queue *q, char *page);
368extern ssize_t blk_throtl_sample_time_store(struct request_queue *q,
369 const char *page, size_t count);
9e234eea 370extern void blk_throtl_bio_endio(struct bio *bio);
b9147dd1 371extern void blk_throtl_stat_add(struct request *rq, u64 time);
9e234eea
SL
372#else
373static inline void blk_throtl_bio_endio(struct bio *bio) { }
b9147dd1 374static inline void blk_throtl_stat_add(struct request *rq, u64 time) { }
297e3d85 375#endif
bc9fcbf9 376
51d798cd 377struct bio *__blk_queue_bounce(struct bio *bio, struct request_queue *q);
9bb33f24
CH
378
379static inline bool blk_queue_may_bounce(struct request_queue *q)
380{
381 return IS_ENABLED(CONFIG_BOUNCE) &&
382 q->limits.bounce == BLK_BOUNCE_HIGH &&
383 max_low_pfn >= max_pfn;
384}
385
51d798cd
CH
386static inline struct bio *blk_queue_bounce(struct bio *bio,
387 struct request_queue *q)
3bce016a 388{
51d798cd
CH
389 if (unlikely(blk_queue_may_bounce(q) && bio_has_data(bio)))
390 return __blk_queue_bounce(bio, q);
391 return bio;
3bce016a 392}
3bce016a 393
bf505456 394#ifdef CONFIG_BLK_DEV_ZONED
5d400665 395void disk_free_zone_bitmaps(struct gendisk *disk);
b3c72f81 396void disk_clear_zone_settings(struct gendisk *disk);
5e4ea834
CH
397int blkdev_report_zones_ioctl(struct block_device *bdev, unsigned int cmd,
398 unsigned long arg);
05bdb996 399int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode,
cfb42576
CH
400 unsigned int cmd, unsigned long arg);
401#else /* CONFIG_BLK_DEV_ZONED */
5d400665 402static inline void disk_free_zone_bitmaps(struct gendisk *disk) {}
b3c72f81 403static inline void disk_clear_zone_settings(struct gendisk *disk) {}
cfb42576 404static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
5e4ea834 405 unsigned int cmd, unsigned long arg)
cfb42576
CH
406{
407 return -ENOTTY;
408}
409static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev,
05bdb996 410 blk_mode_t mode, unsigned int cmd, unsigned long arg)
cfb42576
CH
411{
412 return -ENOTTY;
413}
414#endif /* CONFIG_BLK_DEV_ZONED */
415
416struct block_device *bdev_alloc(struct gendisk *disk, u8 partno);
417void bdev_add(struct block_device *bdev, dev_t dev);
bf505456 418
7c3f828b
CH
419int blk_alloc_ext_minor(void);
420void blk_free_ext_minor(unsigned int minor);
581e2600
CH
421#define ADDPART_FLAG_NONE 0
422#define ADDPART_FLAG_RAID 1
423#define ADDPART_FLAG_WHOLEDISK 2
7f6be376
CH
424int bdev_add_partition(struct gendisk *disk, int partno, sector_t start,
425 sector_t length);
926fbb16 426int bdev_del_partition(struct gendisk *disk, int partno);
3d2e7989
CH
427int bdev_resize_partition(struct gendisk *disk, int partno, sector_t start,
428 sector_t length);
eec1be4c 429void drop_partition(struct block_device *part);
581e2600 430
83794367
DLM
431void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors);
432
6f8191fd
CH
433struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
434 struct lock_class_key *lkclass);
435
e4581105 436int bio_add_hw_page(struct request_queue *q, struct bio *bio,
130879f1 437 struct page *page, unsigned int len, unsigned int offset,
e4581105 438 unsigned int max_sectors, bool *same_page);
130879f1 439
fd363244
DH
440/*
441 * Clean up a page appropriately, where the page may be pinned, may have a
442 * ref taken on it or neither.
443 */
444static inline void bio_release_page(struct bio *bio, struct page *page)
445{
446 if (bio_flagged(bio, BIO_PAGE_PINNED))
447 unpin_user_page(page);
fd363244
DH
448}
449
80bd4a7a 450struct request_queue *blk_alloc_queue(int node_id);
704b914f 451
05bdb996 452int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode);
da7ba729 453
92e7755e 454int disk_alloc_events(struct gendisk *disk);
d5870edf
CH
455void disk_add_events(struct gendisk *disk);
456void disk_del_events(struct gendisk *disk);
457void disk_release_events(struct gendisk *disk);
926597ff
CH
458void disk_block_events(struct gendisk *disk);
459void disk_unblock_events(struct gendisk *disk);
460void disk_flush_events(struct gendisk *disk, unsigned int mask);
2bc8cda5
CH
461extern struct device_attribute dev_attr_events;
462extern struct device_attribute dev_attr_events_async;
463extern struct device_attribute dev_attr_events_poll_msecs;
d5870edf 464
cc5c516d
CH
465extern struct attribute_group blk_trace_attr_group;
466
05bdb996
CH
467blk_mode_t file_to_blk_mode(struct file *file);
468int truncate_bdev_range(struct block_device *bdev, blk_mode_t mode,
469 loff_t lstart, loff_t lend);
8a709512 470long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg);
84b8514b
CH
471long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg);
472
cd82cca7
CH
473extern const struct address_space_operations def_blk_aops;
474
22d0c408 475int disk_register_independent_access_ranges(struct gendisk *disk);
a2247f19
DLM
476void disk_unregister_independent_access_ranges(struct gendisk *disk);
477
06c8c691
CH
478#ifdef CONFIG_FAIL_MAKE_REQUEST
479bool should_fail_request(struct block_device *part, unsigned int bytes);
480#else /* CONFIG_FAIL_MAKE_REQUEST */
481static inline bool should_fail_request(struct block_device *part,
482 unsigned int bytes)
483{
484 return false;
485}
486#endif /* CONFIG_FAIL_MAKE_REQUEST */
487
0a467d0f
JA
488/*
489 * Optimized request reference counting. Ideally we'd make timeouts be more
490 * clever, as that's the only reason we need references at all... But until
491 * this happens, this is faster than using refcount_t. Also see:
492 *
493 * abc54d634334 ("io_uring: switch to atomic_t for io_kiocb reference count")
494 */
495#define req_ref_zero_or_close_to_overflow(req) \
496 ((unsigned int) atomic_read(&(req->ref)) + 127u <= 127u)
497
498static inline bool req_ref_inc_not_zero(struct request *req)
499{
500 return atomic_inc_not_zero(&req->ref);
501}
502
503static inline bool req_ref_put_and_test(struct request *req)
504{
505 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
506 return atomic_dec_and_test(&req->ref);
507}
508
509static inline void req_ref_set(struct request *req, int value)
510{
511 atomic_set(&req->ref, value);
512}
513
514static inline int req_ref_read(struct request *req)
515{
516 return atomic_read(&req->ref);
517}
518
bc9fcbf9 519#endif /* BLK_INTERNAL_H */