Commit | Line | Data |
---|---|---|
8c16567d | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
8fc55455 ML |
2 | /* |
3 | * bvec iterator | |
4 | * | |
5 | * Copyright (C) 2001 Ming Lei <ming.lei@canonical.com> | |
8fc55455 | 6 | */ |
e45cef51 CH |
7 | #ifndef __LINUX_BVEC_H |
8 | #define __LINUX_BVEC_H | |
8fc55455 | 9 | |
e6e74717 | 10 | #include <linux/highmem.h> |
0781e79e | 11 | #include <linux/bug.h> |
b1fb2c52 | 12 | #include <linux/errno.h> |
b296a6d5 AS |
13 | #include <linux/limits.h> |
14 | #include <linux/minmax.h> | |
b296a6d5 AS |
15 | #include <linux/types.h> |
16 | ||
17 | struct page; | |
0781e79e | 18 | |
854b5f01 BVA |
19 | /** |
20 | * struct bio_vec - a contiguous range of physical memory addresses | |
21 | * @bv_page: First page associated with the address range. | |
22 | * @bv_len: Number of bytes in the address range. | |
23 | * @bv_offset: Start of the address range relative to the start of @bv_page. | |
24 | * | |
25 | * The following holds for a bvec if n * PAGE_SIZE < bv_offset + bv_len: | |
26 | * | |
27 | * nth_page(@bv_page, n) == @bv_page + n | |
28 | * | |
29 | * This holds because page_is_mergeable() checks the above property. | |
0781e79e ML |
30 | */ |
31 | struct bio_vec { | |
32 | struct page *bv_page; | |
33 | unsigned int bv_len; | |
34 | unsigned int bv_offset; | |
35 | }; | |
36 | ||
d58cdfae CH |
37 | /** |
38 | * bvec_set_page - initialize a bvec based off a struct page | |
39 | * @bv: bvec to initialize | |
40 | * @page: page the bvec should point to | |
41 | * @len: length of the bvec | |
42 | * @offset: offset into the page | |
43 | */ | |
44 | static inline void bvec_set_page(struct bio_vec *bv, struct page *page, | |
45 | unsigned int len, unsigned int offset) | |
46 | { | |
47 | bv->bv_page = page; | |
48 | bv->bv_len = len; | |
49 | bv->bv_offset = offset; | |
50 | } | |
51 | ||
26db5ee1 CH |
52 | /** |
53 | * bvec_set_folio - initialize a bvec based off a struct folio | |
54 | * @bv: bvec to initialize | |
55 | * @folio: folio the bvec should point to | |
56 | * @len: length of the bvec | |
57 | * @offset: offset into the folio | |
58 | */ | |
59 | static inline void bvec_set_folio(struct bio_vec *bv, struct folio *folio, | |
60 | unsigned int len, unsigned int offset) | |
61 | { | |
62 | bvec_set_page(bv, &folio->page, len, offset); | |
63 | } | |
64 | ||
666e6550 CH |
65 | /** |
66 | * bvec_set_virt - initialize a bvec based on a virtual address | |
67 | * @bv: bvec to initialize | |
68 | * @vaddr: virtual address to set the bvec to | |
69 | * @len: length of the bvec | |
70 | */ | |
71 | static inline void bvec_set_virt(struct bio_vec *bv, void *vaddr, | |
72 | unsigned int len) | |
73 | { | |
74 | bvec_set_page(bv, virt_to_page(vaddr), len, offset_in_page(vaddr)); | |
75 | } | |
76 | ||
0781e79e ML |
77 | struct bvec_iter { |
78 | sector_t bi_sector; /* device address in 512 byte | |
79 | sectors */ | |
80 | unsigned int bi_size; /* residual I/O count */ | |
81 | ||
82 | unsigned int bi_idx; /* current index into bvl_vec */ | |
83 | ||
84 | unsigned int bi_bvec_done; /* number of bytes completed in | |
85 | current bvec */ | |
19416123 | 86 | } __packed; |
8fc55455 | 87 | |
6dc4f100 ML |
88 | struct bvec_iter_all { |
89 | struct bio_vec bv; | |
90 | int idx; | |
91 | unsigned done; | |
92 | }; | |
93 | ||
8fc55455 ML |
94 | /* |
95 | * various member access, note that bio_data should of course not be used | |
96 | * on highmem page vectors | |
97 | */ | |
98 | #define __bvec_iter_bvec(bvec, iter) (&(bvec)[(iter).bi_idx]) | |
99 | ||
3d75ca0a ML |
100 | /* multi-page (mp_bvec) helpers */ |
101 | #define mp_bvec_iter_page(bvec, iter) \ | |
8fc55455 ML |
102 | (__bvec_iter_bvec((bvec), (iter))->bv_page) |
103 | ||
3d75ca0a | 104 | #define mp_bvec_iter_len(bvec, iter) \ |
8fc55455 ML |
105 | min((iter).bi_size, \ |
106 | __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done) | |
107 | ||
3d75ca0a | 108 | #define mp_bvec_iter_offset(bvec, iter) \ |
8fc55455 ML |
109 | (__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done) |
110 | ||
3d75ca0a ML |
111 | #define mp_bvec_iter_page_idx(bvec, iter) \ |
112 | (mp_bvec_iter_offset((bvec), (iter)) / PAGE_SIZE) | |
113 | ||
114 | #define mp_bvec_iter_bvec(bvec, iter) \ | |
115 | ((struct bio_vec) { \ | |
116 | .bv_page = mp_bvec_iter_page((bvec), (iter)), \ | |
117 | .bv_len = mp_bvec_iter_len((bvec), (iter)), \ | |
118 | .bv_offset = mp_bvec_iter_offset((bvec), (iter)), \ | |
119 | }) | |
120 | ||
121 | /* For building single-page bvec in flight */ | |
122 | #define bvec_iter_offset(bvec, iter) \ | |
123 | (mp_bvec_iter_offset((bvec), (iter)) % PAGE_SIZE) | |
124 | ||
125 | #define bvec_iter_len(bvec, iter) \ | |
126 | min_t(unsigned, mp_bvec_iter_len((bvec), (iter)), \ | |
127 | PAGE_SIZE - bvec_iter_offset((bvec), (iter))) | |
128 | ||
129 | #define bvec_iter_page(bvec, iter) \ | |
52d52d1c CH |
130 | (mp_bvec_iter_page((bvec), (iter)) + \ |
131 | mp_bvec_iter_page_idx((bvec), (iter))) | |
3d75ca0a | 132 | |
8fc55455 ML |
133 | #define bvec_iter_bvec(bvec, iter) \ |
134 | ((struct bio_vec) { \ | |
135 | .bv_page = bvec_iter_page((bvec), (iter)), \ | |
136 | .bv_len = bvec_iter_len((bvec), (iter)), \ | |
137 | .bv_offset = bvec_iter_offset((bvec), (iter)), \ | |
138 | }) | |
139 | ||
b1fb2c52 DM |
140 | static inline bool bvec_iter_advance(const struct bio_vec *bv, |
141 | struct bvec_iter *iter, unsigned bytes) | |
8fc55455 | 142 | { |
795ee49c PB |
143 | unsigned int idx = iter->bi_idx; |
144 | ||
b1fb2c52 DM |
145 | if (WARN_ONCE(bytes > iter->bi_size, |
146 | "Attempted to advance past end of bvec iter\n")) { | |
147 | iter->bi_size = 0; | |
148 | return false; | |
149 | } | |
8fc55455 | 150 | |
795ee49c PB |
151 | iter->bi_size -= bytes; |
152 | bytes += iter->bi_bvec_done; | |
8fc55455 | 153 | |
795ee49c PB |
154 | while (bytes && bytes >= bv[idx].bv_len) { |
155 | bytes -= bv[idx].bv_len; | |
156 | idx++; | |
8fc55455 | 157 | } |
795ee49c PB |
158 | |
159 | iter->bi_idx = idx; | |
160 | iter->bi_bvec_done = bytes; | |
b1fb2c52 | 161 | return true; |
8fc55455 ML |
162 | } |
163 | ||
6b6667aa PB |
164 | /* |
165 | * A simpler version of bvec_iter_advance(), @bytes should not span | |
166 | * across multiple bvec entries, i.e. bytes <= bv[i->bi_idx].bv_len | |
167 | */ | |
168 | static inline void bvec_iter_advance_single(const struct bio_vec *bv, | |
169 | struct bvec_iter *iter, unsigned int bytes) | |
7e249690 | 170 | { |
6b6667aa PB |
171 | unsigned int done = iter->bi_bvec_done + bytes; |
172 | ||
173 | if (done == bv[iter->bi_idx].bv_len) { | |
174 | done = 0; | |
175 | iter->bi_idx++; | |
176 | } | |
177 | iter->bi_bvec_done = done; | |
178 | iter->bi_size -= bytes; | |
7e249690 ML |
179 | } |
180 | ||
8fc55455 ML |
181 | #define for_each_bvec(bvl, bio_vec, iter, start) \ |
182 | for (iter = (start); \ | |
183 | (iter).bi_size && \ | |
184 | ((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \ | |
6b6667aa | 185 | bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len)) |
8fc55455 | 186 | |
3c892a09 ML |
187 | /* for iterating one bio from start to end */ |
188 | #define BVEC_ITER_ALL_INIT (struct bvec_iter) \ | |
189 | { \ | |
190 | .bi_sector = 0, \ | |
191 | .bi_size = UINT_MAX, \ | |
192 | .bi_idx = 0, \ | |
193 | .bi_bvec_done = 0, \ | |
194 | } | |
195 | ||
6dc4f100 ML |
196 | static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all) |
197 | { | |
6dc4f100 | 198 | iter_all->done = 0; |
1200e07f | 199 | iter_all->idx = 0; |
6dc4f100 ML |
200 | |
201 | return &iter_all->bv; | |
202 | } | |
203 | ||
1200e07f ML |
204 | static inline void bvec_advance(const struct bio_vec *bvec, |
205 | struct bvec_iter_all *iter_all) | |
6dc4f100 ML |
206 | { |
207 | struct bio_vec *bv = &iter_all->bv; | |
208 | ||
1200e07f | 209 | if (iter_all->done) { |
52d52d1c | 210 | bv->bv_page++; |
6dc4f100 ML |
211 | bv->bv_offset = 0; |
212 | } else { | |
b8753433 | 213 | bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT); |
6bedf00e | 214 | bv->bv_offset = bvec->bv_offset & ~PAGE_MASK; |
6dc4f100 ML |
215 | } |
216 | bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset, | |
217 | bvec->bv_len - iter_all->done); | |
1200e07f ML |
218 | iter_all->done += bv->bv_len; |
219 | ||
220 | if (iter_all->done == bvec->bv_len) { | |
221 | iter_all->idx++; | |
222 | iter_all->done = 0; | |
223 | } | |
6dc4f100 ML |
224 | } |
225 | ||
e6e74717 CH |
226 | /** |
227 | * bvec_kmap_local - map a bvec into the kernel virtual address space | |
228 | * @bvec: bvec to map | |
229 | * | |
230 | * Must be called on single-page bvecs only. Call kunmap_local on the returned | |
231 | * address to unmap. | |
232 | */ | |
233 | static inline void *bvec_kmap_local(struct bio_vec *bvec) | |
234 | { | |
235 | return kmap_local_page(bvec->bv_page) + bvec->bv_offset; | |
236 | } | |
237 | ||
f93a181a CH |
238 | /** |
239 | * memcpy_from_bvec - copy data from a bvec | |
240 | * @bvec: bvec to copy from | |
241 | * | |
242 | * Must be called on single-page bvecs only. | |
243 | */ | |
244 | static inline void memcpy_from_bvec(char *to, struct bio_vec *bvec) | |
245 | { | |
246 | memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, bvec->bv_len); | |
247 | } | |
248 | ||
249 | /** | |
250 | * memcpy_to_bvec - copy data to a bvec | |
251 | * @bvec: bvec to copy to | |
252 | * | |
253 | * Must be called on single-page bvecs only. | |
254 | */ | |
255 | static inline void memcpy_to_bvec(struct bio_vec *bvec, const char *from) | |
256 | { | |
257 | memcpy_to_page(bvec->bv_page, bvec->bv_offset, from, bvec->bv_len); | |
258 | } | |
259 | ||
260 | /** | |
261 | * memzero_bvec - zero all data in a bvec | |
262 | * @bvec: bvec to zero | |
263 | * | |
264 | * Must be called on single-page bvecs only. | |
265 | */ | |
266 | static inline void memzero_bvec(struct bio_vec *bvec) | |
267 | { | |
268 | memzero_page(bvec->bv_page, bvec->bv_offset, bvec->bv_len); | |
269 | } | |
270 | ||
1113f0b6 CH |
271 | /** |
272 | * bvec_virt - return the virtual address for a bvec | |
273 | * @bvec: bvec to return the virtual address for | |
274 | * | |
275 | * Note: the caller must ensure that @bvec->bv_page is not a highmem page. | |
276 | */ | |
277 | static inline void *bvec_virt(struct bio_vec *bvec) | |
278 | { | |
279 | WARN_ON_ONCE(PageHighMem(bvec->bv_page)); | |
280 | return page_address(bvec->bv_page) + bvec->bv_offset; | |
281 | } | |
282 | ||
e45cef51 | 283 | #endif /* __LINUX_BVEC_H */ |