iov_iter: don't call into iov_iter_revert() for non-zero bytes

Most callers don't check if there's actually data to revert, so we end
up doing a function call only to return immediately because unroll is
equal to zero.

Add an inline helper that checks if we have bytes to unroll, and only
call into __iov_iter_revert() if that is the case, rather than add
this check in the callers we care about.

This saves 0.30% for me in workload that uses iov iterators intensively.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/uio.h b/include/linux/uio.h
index bea9c89922d908f66511dacb02edc8f4bcad918c..e140c27fdd352051b774ca54ebbb80c935dec687 100644 (file)
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -162,7 +162,12 @@ static inline size_t iov_length(const struct iovec *iov, unsigned long nr_segs)
 size_t copy_page_from_iter_atomic(struct page *page, size_t offset,
                                  size_t bytes, struct iov_iter *i);
 void iov_iter_advance(struct iov_iter *i, size_t bytes);
-void iov_iter_revert(struct iov_iter *i, size_t bytes);
+void __iov_iter_revert(struct iov_iter *i, size_t bytes);
+static inline void iov_iter_revert(struct iov_iter *i, size_t bytes)
+{
+       if (bytes)
+               __iov_iter_revert(i, bytes);
+}
 size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t bytes);
 size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t bytes);
 size_t iov_iter_single_seg_count(const struct iov_iter *i);

diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 5e34639c5d1e614cfd5d3bd3c1c16791f16e23fd..4d81ce122e3c39a501e46c0e9416d31bb1eabf12 100644 (file)
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -564,10 +564,8 @@ void iov_iter_advance(struct iov_iter *i, size_t size)
 }
 EXPORT_SYMBOL(iov_iter_advance);
 
-void iov_iter_revert(struct iov_iter *i, size_t unroll)
+void __iov_iter_revert(struct iov_iter *i, size_t unroll)
 {
-       if (!unroll)
-               return;
        if (WARN_ON(unroll > MAX_RW_COUNT))
                return;
        i->count += unroll;
@@ -609,7 +607,7 @@ void iov_iter_revert(struct iov_iter *i, size_t unroll)
                }
        }
 }
-EXPORT_SYMBOL(iov_iter_revert);
+EXPORT_SYMBOL(__iov_iter_revert);
 
 /*
  * Return the count of just the current iov_iter segment.