Commit | Line | Data |
---|---|---|
61cbe788 ML |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* | |
3 | * Copyright 2022 Red Hat, Inc. | |
4 | */ | |
5 | ||
6 | #include <linux/bio.h> | |
7 | #include <linux/blk-crypto.h> | |
8 | #include <linux/blk-integrity.h> | |
9 | ||
10 | #include "dm-core.h" | |
11 | ||
12 | static inline bool dm_bvec_iter_rewind(const struct bio_vec *bv, | |
13 | struct bvec_iter *iter, | |
14 | unsigned int bytes) | |
15 | { | |
16 | int idx; | |
17 | ||
18 | iter->bi_size += bytes; | |
19 | if (bytes <= iter->bi_bvec_done) { | |
20 | iter->bi_bvec_done -= bytes; | |
21 | return true; | |
22 | } | |
23 | ||
24 | bytes -= iter->bi_bvec_done; | |
25 | idx = iter->bi_idx - 1; | |
26 | ||
27 | while (idx >= 0 && bytes && bytes > bv[idx].bv_len) { | |
28 | bytes -= bv[idx].bv_len; | |
29 | idx--; | |
30 | } | |
31 | ||
32 | if (WARN_ONCE(idx < 0 && bytes, | |
33 | "Attempted to rewind iter beyond bvec's boundaries\n")) { | |
34 | iter->bi_size -= bytes; | |
35 | iter->bi_bvec_done = 0; | |
36 | iter->bi_idx = 0; | |
37 | return false; | |
38 | } | |
39 | ||
40 | iter->bi_idx = idx; | |
41 | iter->bi_bvec_done = bv[idx].bv_len - bytes; | |
42 | return true; | |
43 | } | |
44 | ||
45 | #if defined(CONFIG_BLK_DEV_INTEGRITY) | |
46 | ||
47 | /** | |
48 | * dm_bio_integrity_rewind - Rewind integrity vector | |
49 | * @bio: bio whose integrity vector to update | |
50 | * @bytes_done: number of data bytes to rewind | |
51 | * | |
52 | * Description: This function calculates how many integrity bytes the | |
53 | * number of completed data bytes correspond to and rewind the | |
54 | * integrity vector accordingly. | |
55 | */ | |
56 | static void dm_bio_integrity_rewind(struct bio *bio, unsigned int bytes_done) | |
57 | { | |
58 | struct bio_integrity_payload *bip = bio_integrity(bio); | |
59 | struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); | |
60 | unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9); | |
61 | ||
62 | bip->bip_iter.bi_sector -= bio_integrity_intervals(bi, bytes_done >> 9); | |
63 | dm_bvec_iter_rewind(bip->bip_vec, &bip->bip_iter, bytes); | |
64 | } | |
65 | ||
66 | #else /* CONFIG_BLK_DEV_INTEGRITY */ | |
67 | ||
68 | static inline void dm_bio_integrity_rewind(struct bio *bio, | |
69 | unsigned int bytes_done) | |
70 | { | |
71 | return; | |
72 | } | |
73 | ||
74 | #endif | |
75 | ||
76 | #if defined(CONFIG_BLK_INLINE_ENCRYPTION) | |
77 | ||
78 | /* Decrements @dun by @dec, treating @dun as a multi-limb integer. */ | |
79 | static void dm_bio_crypt_dun_decrement(u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], | |
80 | unsigned int dec) | |
81 | { | |
82 | int i; | |
83 | ||
84 | for (i = 0; dec && i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) { | |
85 | u64 prev = dun[i]; | |
86 | ||
87 | dun[i] -= dec; | |
88 | if (dun[i] > prev) | |
89 | dec = 1; | |
90 | else | |
91 | dec = 0; | |
92 | } | |
93 | } | |
94 | ||
95 | static void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes) | |
96 | { | |
97 | struct bio_crypt_ctx *bc = bio->bi_crypt_context; | |
98 | ||
99 | dm_bio_crypt_dun_decrement(bc->bc_dun, | |
100 | bytes >> bc->bc_key->data_unit_size_bits); | |
101 | } | |
102 | ||
103 | #else /* CONFIG_BLK_INLINE_ENCRYPTION */ | |
104 | ||
105 | static inline void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes) | |
106 | { | |
107 | return; | |
108 | } | |
109 | ||
110 | #endif | |
111 | ||
112 | static inline void dm_bio_rewind_iter(const struct bio *bio, | |
113 | struct bvec_iter *iter, unsigned int bytes) | |
114 | { | |
115 | iter->bi_sector -= bytes >> 9; | |
116 | ||
117 | /* No advance means no rewind */ | |
118 | if (bio_no_advance_iter(bio)) | |
119 | iter->bi_size += bytes; | |
120 | else | |
121 | dm_bvec_iter_rewind(bio->bi_io_vec, iter, bytes); | |
122 | } | |
123 | ||
124 | /** | |
125 | * dm_bio_rewind - update ->bi_iter of @bio by rewinding @bytes. | |
126 | * @bio: bio to rewind | |
127 | * @bytes: how many bytes to rewind | |
128 | * | |
129 | * WARNING: | |
130 | * Caller must ensure that @bio has a fixed end sector, to allow | |
131 | * rewinding from end of bio and restoring its original position. | |
132 | * Caller is also responsibile for restoring bio's size. | |
133 | */ | |
8b211aac | 134 | static void dm_bio_rewind(struct bio *bio, unsigned bytes) |
61cbe788 ML |
135 | { |
136 | if (bio_integrity(bio)) | |
137 | dm_bio_integrity_rewind(bio, bytes); | |
138 | ||
139 | if (bio_has_crypt_ctx(bio)) | |
140 | dm_bio_crypt_rewind(bio, bytes); | |
141 | ||
142 | dm_bio_rewind_iter(bio, &bio->bi_iter, bytes); | |
143 | } | |
8b211aac ML |
144 | |
145 | void dm_io_rewind(struct dm_io *io, struct bio_set *bs) | |
146 | { | |
147 | struct bio *orig = io->orig_bio; | |
148 | struct bio *new_orig = bio_alloc_clone(orig->bi_bdev, orig, | |
149 | GFP_NOIO, bs); | |
150 | /* | |
151 | * dm_bio_rewind can restore to previous position since the | |
152 | * end sector is fixed for original bio, but we still need | |
153 | * to restore bio's size manually (using io->sectors). | |
154 | */ | |
155 | dm_bio_rewind(new_orig, ((io->sector_offset << 9) - | |
156 | orig->bi_iter.bi_size)); | |
157 | bio_trim(new_orig, 0, io->sectors); | |
158 | ||
159 | bio_chain(new_orig, orig); | |
160 | /* | |
161 | * __bi_remaining was increased (by dm_split_and_process_bio), | |
162 | * so must drop the one added in bio_chain. | |
163 | */ | |
164 | atomic_dec(&orig->__bi_remaining); | |
165 | io->orig_bio = new_orig; | |
166 | } |