block: split bio_queue_enter from blk_queue_enter

To prepare for fixing a gendisk shutdown race, open code the
blk_queue_enter logic in bio_queue_enter.  This also removes the
pointless flags translation.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Tested-by: Darrick J. Wong <djwong@kernel.org>
Link: https://lore.kernel.org/r/20210929071241.934472-4-hch@lst.de
Tested-by: Yi Zhang <yi.zhang@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-core.c b/block/blk-core.c
index 7e9eadacf2dea2d53ac732faa1400f2a26afebec..43f5da707d8e3286a8752e8e095e65f717f45566 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -475,18 +475,35 @@ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
 static inline int bio_queue_enter(struct bio *bio)
 {
        struct request_queue *q = bio->bi_bdev->bd_disk->queue;
-       bool nowait = bio->bi_opf & REQ_NOWAIT;
-       int ret;
 
-       ret = blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0);
-       if (unlikely(ret)) {
-               if (nowait && !blk_queue_dying(q))
+       while (!blk_try_enter_queue(q, false)) {
+               if (bio->bi_opf & REQ_NOWAIT) {
+                       if (blk_queue_dying(q))
+                               goto dead;
                        bio_wouldblock_error(bio);
-               else
-                       bio_io_error(bio);
+                       return -EBUSY;
+               }
+
+               /*
+                * read pair of barrier in blk_freeze_queue_start(), we need to
+                * order reading __PERCPU_REF_DEAD flag of .q_usage_counter and
+                * reading .mq_freeze_depth or queue dying flag, otherwise the
+                * following wait may never return if the two reads are
+                * reordered.
+                */
+               smp_rmb();
+               wait_event(q->mq_freeze_wq,
+                          (!q->mq_freeze_depth &&
+                           blk_pm_resume_queue(false, q)) ||
+                          blk_queue_dying(q));
+               if (blk_queue_dying(q))
+                       goto dead;
        }
 
-       return ret;
+       return 0;
+dead:
+       bio_io_error(bio);
+       return -ENODEV;
 }
 
 void blk_queue_exit(struct request_queue *q)