summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristoph Hellwig <hch@lst.de>2016-11-16 17:35:34 +0100
committerJens Axboe <axboe@fb.com>2016-11-16 23:33:10 -0700
commit7b3127b98b3c249f0c9e01a8cdcb5da04fc59d9a (patch)
tree87b21b1e16bb0ac01b6d437535b7e3ec7fb92e37
parent39ebe60daa659cc111d279a84b48ffb71a5c6786 (diff)
block_dev: avoid atomic ops in direct I/O unless necessary
Signed-off-by: Christoph Hellwig <hch@lst.de> Signed-off-by: Jens Axboe <axboe@fb.com>
-rw-r--r--fs/block_dev.c13
1 files changed, 10 insertions, 3 deletions
diff --git a/fs/block_dev.c b/fs/block_dev.c
index 176f388e0ccd..e2ae7a696d75 100644
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -265,6 +265,7 @@ struct blkdev_dio {
struct task_struct *waiter;
size_t size;
atomic_t ref;
+ bool multi_bio : 1;
bool should_dirty : 1;
struct bio bio;
};
@@ -276,7 +277,7 @@ static void blkdev_bio_end_io(struct bio *bio)
struct blkdev_dio *dio = bio->bi_private;
struct kiocb *iocb = dio->iocb;
- if (!atomic_dec_and_test(&dio->ref)) {
+ if (dio->multi_bio && !atomic_dec_and_test(&dio->ref)) {
if (bio->bi_error && !dio->bio.bi_error)
dio->bio.bi_error = bio->bi_error;
} else {
@@ -335,7 +336,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
dio->iocb = is_sync ? NULL : iocb;
dio->waiter = current;
dio->size = 0;
- atomic_set(&dio->ref, 1);
+ dio->multi_bio = false;
dio->should_dirty = is_read && (iter->type == ITER_IOVEC);
for (;;) {
@@ -375,7 +376,13 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
break;
}
- atomic_inc(&dio->ref);
+ if (!dio->multi_bio) {
+ dio->multi_bio = true;
+ atomic_set(&dio->ref, 2);
+ } else {
+ atomic_inc(&dio->ref);
+ }
+
submit_bio(bio);
bio = bio_alloc(GFP_KERNEL, nr_pages);
}