direct-io: only inc/dec inode->i_dio_count for file systems

We don't need truncate protection for block devices, so add a flag
bypassing this cache line dirtying twice for every IO. This easily
contributes to 5-10% of the CPU time on high IOPS O_DIRECT testing.

Cc: Christoph Hellwig <hch@infradead.org>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: akpm@linux-foundation.org
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/block_dev.c b/fs/block_dev.c
index 1e86823a9cbda37f8451269d91a50f90d00c9566..e20b7c1dcbae0a228f378677b5503797c8dbc9bd 100644 (file)
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -172,7 +172,8 @@ blkdev_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
        struct inode *inode = file->f_mapping->host;
 
        return __blockdev_direct_IO(rw, iocb, inode, I_BDEV(inode), iov, offset,
-                                   nr_segs, blkdev_get_block, NULL, NULL, 0);
+                                   nr_segs, blkdev_get_block, NULL, NULL,
+                                   DIO_IGNORE_TRUNCATE);
 }
 
 int __sync_blockdev(struct block_device *bdev, int wait)

diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index b0ef7b07b1b3be510e85dd1bab6711e7d27cebee..2d51e12483646fdaababd33d4be05d160e1fb868 100644 (file)
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -7238,7 +7238,7 @@ static ssize_t btrfs_direct_IO(int rw, struct kiocb *iocb,
                        goto out;
        } else if (unlikely(test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
                                     &BTRFS_I(inode)->runtime_flags))) {
-               inode_dio_done(inode);
+               inode_dio_done(inode, 0);
                flags = DIO_LOCKING | DIO_SKIP_HOLES;
                wakeup = false;
        }
@@ -7258,7 +7258,7 @@ static ssize_t btrfs_direct_IO(int rw, struct kiocb *iocb,
        }
 out:
        if (wakeup)
-               inode_dio_done(inode);
+               inode_dio_done(inode, 0);
        if (relock)
                mutex_lock(&inode->i_mutex);
 

diff --git a/fs/direct-io.c b/fs/direct-io.c
index 0e04142d5962312fcb055738479247b2364a252e..5aa297b69003c27b70ddafc6a7b937645b1da84b 100644 (file)
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -265,7 +265,8 @@ static ssize_t dio_complete(struct dio *dio, loff_t offset, ssize_t ret,
        if (dio->end_io && dio->result)
                dio->end_io(dio->iocb, offset, transferred, dio->private);
 
-       inode_dio_done(dio->inode);
+       inode_dio_done(dio->inode, dio->flags);
+
        if (is_async) {
                if (dio->rw & WRITE) {
                        int err;
@@ -1222,10 +1223,7 @@ do_blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
                }
        }
 
-       /*
-        * Will be decremented at I/O completion time.
-        */
-       atomic_inc(&inode->i_dio_count);
+       inode_dio_start(inode, dio->flags);
 
        retval = 0;
        sdio.blkbits = blkbits;

diff --git a/fs/ext4/indirect.c b/fs/ext4/indirect.c
index 594009f5f523f0fd2228a72f1aa593a6f3ebf66f..5dc9fadd36ea4b39e123f283aed1d67f29dc2734 100644 (file)
--- a/fs/ext4/indirect.c
+++ b/fs/ext4/indirect.c
@@ -683,14 +683,14 @@ retry:
                smp_mb();
                if (unlikely(ext4_test_inode_state(inode,
                                                    EXT4_STATE_DIOREAD_LOCK))) {
-                       inode_dio_done(inode);
+                       inode_dio_done(inode, 0);
                        goto locked;
                }
                ret = __blockdev_direct_IO(rw, iocb, inode,
                                 inode->i_sb->s_bdev, iov,
                                 offset, nr_segs,
                                 ext4_get_block, NULL, NULL, 0);
-               inode_dio_done(inode);
+               inode_dio_done(inode, 0);
        } else {
 locked:
                ret = blockdev_direct_IO(rw, iocb, inode, iov,

diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index e274e9c1171f9095829aff07224dfff650a85ed2..87b8c29b9f275420d2b50b2780f6ff14d2193afe 100644 (file)
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -3193,7 +3193,7 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
 
 retake_lock:
        if (rw == WRITE)
-               inode_dio_done(inode);
+               inode_dio_done(inode, 0);
        /* take i_mutex locking again if we do a ovewrite dio */
        if (overwrite) {
                up_read(&EXT4_I(inode)->i_data_sem);

diff --git a/fs/inode.c b/fs/inode.c
index b33ba8e021cc286d500d94abd64d8f85115ebcd2..2a1819cd8a0258a114be035f89f935afae055770 100644 (file)
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -1854,6 +1854,20 @@ void inode_dio_wait(struct inode *inode)
 }
 EXPORT_SYMBOL(inode_dio_wait);
 
+void inode_dio_start(struct inode *inode, int dio_flags)
+{
+       if (!(dio_flags & DIO_IGNORE_TRUNCATE))
+               atomic_inc(&inode->i_dio_count);
+}
+
+static int inode_dio_dec(struct inode *inode, int dio_flags)
+{
+       if (!(dio_flags & DIO_IGNORE_TRUNCATE))
+               return atomic_dec_and_test(&inode->i_dio_count);
+
+       return 0;
+}
+
 /*
  * inode_dio_done - signal finish of a direct I/O requests
  * @inode: inode the direct I/O happens on
@@ -1861,9 +1875,9 @@ EXPORT_SYMBOL(inode_dio_wait);
  * This is called once we've finished processing a direct I/O request,
  * and is used to wake up callers waiting for direct I/O to be quiesced.
  */
-void inode_dio_done(struct inode *inode)
+void inode_dio_done(struct inode *inode, int dio_flags)
 {
-       if (atomic_dec_and_test(&inode->i_dio_count))
+       if (inode_dio_dec(inode, dio_flags))
                wake_up_bit(&inode->i_state, __I_DIO_WAKEUP);
 }
 EXPORT_SYMBOL(inode_dio_done);

diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c
index 91ff089d34126d8ae5d8d4a96bca232fbd921958..8bfa04b4504c8b1138aea849ba7a773aa4b59dcd 100644 (file)
--- a/fs/nfs/direct.c
+++ b/fs/nfs/direct.c
@@ -476,7 +476,7 @@ out:
 static void nfs_inode_dio_write_done(struct inode *inode)
 {
        nfs_zap_mapping(inode, inode->i_mapping);
-       inode_dio_done(inode);
+       inode_dio_done(inode, 0);
 }
 
 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
@@ -833,7 +833,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
         * generic layer handle the completion.
         */
        if (requested_bytes == 0) {
-               inode_dio_done(inode);
+               inode_dio_done(inode, 0);
                nfs_direct_req_release(dreq);
                return result < 0 ? result : -EIO;
        }

diff --git a/include/linux/fs.h b/include/linux/fs.h
index 3f40547ba1917cd038f085bcdb6c6e577a9d4538..a1c23f8a874df6991f0deafd3dc104a9c951d4a6 100644 (file)
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -2468,6 +2468,9 @@ enum {
 
        /* filesystem does not support filling holes */
        DIO_SKIP_HOLES  = 0x02,
+
+       /* inode/fs/bdev does not need truncate protection */
+       DIO_IGNORE_TRUNCATE = 0x04,
 };
 
 void dio_end_io(struct bio *bio, int error);
@@ -2488,7 +2491,8 @@ static inline ssize_t blockdev_direct_IO(int rw, struct kiocb *iocb,
 #endif
 
 void inode_dio_wait(struct inode *inode);
-void inode_dio_done(struct inode *inode);
+void inode_dio_done(struct inode *inode, int dio_flags);
+void inode_dio_start(struct inode *inode, int dio_flags);
 
 extern const struct file_operations generic_ro_fops;