bio: optimise bvec iteration

[linux-block.git] / include / linux / bio.h

diff --git a/include/linux/bio.h b/include/linux/bio.h
index ecf67108f091d768a676a8176b9e4d942eee2ba2..1edda614f7ce2b766b4c6ed9206d5878d0dc2b89 100644 (file)
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -148,11 +148,24 @@ static inline void bio_advance_iter(const struct bio *bio,
                /* TODO: It is reasonable to complete bio with error here. */
 }
 
+/* @bytes should be less or equal to bvec[i->bi_idx].bv_len */
+static inline void bio_advance_iter_single(const struct bio *bio,
+                                          struct bvec_iter *iter,
+                                          unsigned int bytes)
+{
+       iter->bi_sector += bytes >> 9;
+
+       if (bio_no_advance_iter(bio))
+               iter->bi_size -= bytes;
+       else
+               bvec_iter_advance_single(bio->bi_io_vec, iter, bytes);
+}
+
 #define __bio_for_each_segment(bvl, bio, iter, start)                  \
        for (iter = (start);                                            \
             (iter).bi_size &&                                          \
                ((bvl = bio_iter_iovec((bio), (iter))), 1);             \
-            bio_advance_iter((bio), &(iter), (bvl).bv_len))
+            bio_advance_iter_single((bio), &(iter), (bvl).bv_len))
 
 #define bio_for_each_segment(bvl, bio, iter)                           \
        __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
@@ -161,7 +174,7 @@ static inline void bio_advance_iter(const struct bio *bio,
        for (iter = (start);                                            \
             (iter).bi_size &&                                          \
                ((bvl = mp_bvec_iter_bvec((bio)->bi_io_vec, (iter))), 1); \
-            bio_advance_iter((bio), &(iter), (bvl).bv_len))
+            bio_advance_iter_single((bio), &(iter), (bvl).bv_len))
 
 /* iterate over multi-page bvec */
 #define bio_for_each_bvec(bvl, bio, iter)                      \