Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
199a31c6 CH |
2 | #ifndef LINUX_IOMAP_H |
3 | #define LINUX_IOMAP_H 1 | |
4 | ||
9dc55f13 CH |
5 | #include <linux/atomic.h> |
6 | #include <linux/bitmap.h> | |
598ecfba | 7 | #include <linux/blk_types.h> |
9dc55f13 | 8 | #include <linux/mm.h> |
199a31c6 | 9 | #include <linux/types.h> |
5780a02f | 10 | #include <linux/mm_types.h> |
db074436 | 11 | #include <linux/blkdev.h> |
199a31c6 | 12 | |
89eb1906 | 13 | struct address_space; |
8be9f564 | 14 | struct fiemap_extent_info; |
ae259a9c | 15 | struct inode; |
9060bc4d | 16 | struct iomap_iter; |
c3d4ed1a | 17 | struct iomap_dio; |
598ecfba | 18 | struct iomap_writepage_ctx; |
ae259a9c CH |
19 | struct iov_iter; |
20 | struct kiocb; | |
63899c6f | 21 | struct page; |
ae259a9c CH |
22 | struct vm_area_struct; |
23 | struct vm_fault; | |
24 | ||
25 | /* | |
26 | * Types of block ranges for iomap mappings: | |
27 | */ | |
eb81cf9d CH |
28 | #define IOMAP_HOLE 0 /* no blocks allocated, need allocation */ |
29 | #define IOMAP_DELALLOC 1 /* delayed allocation blocks */ | |
30 | #define IOMAP_MAPPED 2 /* blocks allocated at @addr */ | |
31 | #define IOMAP_UNWRITTEN 3 /* blocks allocated at @addr in unwritten state */ | |
32 | #define IOMAP_INLINE 4 /* data inline in the inode */ | |
199a31c6 | 33 | |
17de0a9f | 34 | /* |
65a60e86 CH |
35 | * Flags reported by the file system from iomap_begin: |
36 | * | |
37 | * IOMAP_F_NEW indicates that the blocks have been newly allocated and need | |
38 | * zeroing for areas that no data is copied to. | |
a3841f94 | 39 | * |
caa51d26 JK |
40 | * IOMAP_F_DIRTY indicates the inode has uncommitted metadata needed to access |
41 | * written data and requires fdatasync to commit them to persistent storage. | |
7684e2c4 DC |
42 | * This needs to take into account metadata changes that *may* be made at IO |
43 | * completion, such as file size updates from direct IO. | |
65a60e86 CH |
44 | * |
45 | * IOMAP_F_SHARED indicates that the blocks are shared, and will need to be | |
46 | * unshared as part a write. | |
47 | * | |
48 | * IOMAP_F_MERGED indicates that the iomap contains the merge of multiple block | |
49 | * mappings. | |
50 | * | |
51 | * IOMAP_F_BUFFER_HEAD indicates that the file system requires the use of | |
52 | * buffer heads for this mapping. | |
d7b64041 DC |
53 | * |
54 | * IOMAP_F_XATTR indicates that the iomap is for an extended attribute extent | |
55 | * rather than a file data extent. | |
17de0a9f | 56 | */ |
d7b64041 DC |
57 | #define IOMAP_F_NEW (1U << 0) |
58 | #define IOMAP_F_DIRTY (1U << 1) | |
59 | #define IOMAP_F_SHARED (1U << 2) | |
60 | #define IOMAP_F_MERGED (1U << 3) | |
61 | #define IOMAP_F_BUFFER_HEAD (1U << 4) | |
8e81aa16 | 62 | #define IOMAP_F_XATTR (1U << 5) |
d33fd776 CH |
63 | |
64 | /* | |
65a60e86 CH |
65 | * Flags set by the core iomap code during operations: |
66 | * | |
67 | * IOMAP_F_SIZE_CHANGED indicates to the iomap_end method that the file size | |
68 | * has changed as the result of this write operation. | |
d7b64041 DC |
69 | * |
70 | * IOMAP_F_STALE indicates that the iomap is not valid any longer and the file | |
71 | * range it covers needs to be remapped by the high level before the operation | |
72 | * can proceed. | |
d33fd776 | 73 | */ |
d7b64041 DC |
74 | #define IOMAP_F_SIZE_CHANGED (1U << 8) |
75 | #define IOMAP_F_STALE (1U << 9) | |
17de0a9f | 76 | |
7ee66c03 CH |
77 | /* |
78 | * Flags from 0x1000 up are for file system specific usage: | |
79 | */ | |
d7b64041 | 80 | #define IOMAP_F_PRIVATE (1U << 12) |
7ee66c03 CH |
81 | |
82 | ||
ae259a9c | 83 | /* |
19fe5f64 | 84 | * Magic value for addr: |
ae259a9c | 85 | */ |
19fe5f64 | 86 | #define IOMAP_NULL_ADDR -1ULL /* addr is not valid */ |
199a31c6 | 87 | |
471859f5 | 88 | struct iomap_folio_ops; |
df0db3ec | 89 | |
199a31c6 | 90 | struct iomap { |
19fe5f64 | 91 | u64 addr; /* disk offset of mapping, bytes */ |
ae259a9c CH |
92 | loff_t offset; /* file offset of mapping, bytes */ |
93 | u64 length; /* length of mapping, bytes */ | |
17de0a9f CH |
94 | u16 type; /* type of mapping */ |
95 | u16 flags; /* flags for mapping */ | |
ae259a9c | 96 | struct block_device *bdev; /* block device for I/O */ |
fa5d932c | 97 | struct dax_device *dax_dev; /* dax_dev for dax operations */ |
19e0c58f | 98 | void *inline_data; |
e184fde6 | 99 | void *private; /* filesystem private */ |
471859f5 | 100 | const struct iomap_folio_ops *folio_ops; |
d7b64041 | 101 | u64 validity_cookie; /* used with .iomap_valid() */ |
df0db3ec | 102 | }; |
63899c6f | 103 | |
66b8165e | 104 | static inline sector_t iomap_sector(const struct iomap *iomap, loff_t pos) |
db074436 DW |
105 | { |
106 | return (iomap->addr + pos - iomap->offset) >> SECTOR_SHIFT; | |
107 | } | |
108 | ||
69f4a26c GX |
109 | /* |
110 | * Returns the inline data pointer for logical offset @pos. | |
111 | */ | |
4495c33e | 112 | static inline void *iomap_inline_data(const struct iomap *iomap, loff_t pos) |
69f4a26c GX |
113 | { |
114 | return iomap->inline_data + pos - iomap->offset; | |
115 | } | |
116 | ||
117 | /* | |
118 | * Check if the mapping's length is within the valid range for inline data. | |
119 | * This is used to guard against accessing data beyond the page inline_data | |
120 | * points at. | |
121 | */ | |
e3c4ffb0 | 122 | static inline bool iomap_inline_data_valid(const struct iomap *iomap) |
69f4a26c GX |
123 | { |
124 | return iomap->length <= PAGE_SIZE - offset_in_page(iomap->inline_data); | |
125 | } | |
126 | ||
df0db3ec | 127 | /* |
471859f5 | 128 | * When a filesystem sets folio_ops in an iomap mapping it returns, get_folio |
40405ddd AG |
129 | * and put_folio will be called for each folio written to. This only applies |
130 | * to buffered writes as unbuffered writes will not typically have folios | |
df0db3ec AG |
131 | * associated with them. |
132 | * | |
c82abc23 | 133 | * When get_folio succeeds, put_folio will always be called to do any |
9060bc4d AG |
134 | * cleanup work necessary. put_folio is responsible for unlocking and putting |
135 | * @folio. | |
df0db3ec | 136 | */ |
471859f5 | 137 | struct iomap_folio_ops { |
c82abc23 | 138 | struct folio *(*get_folio)(struct iomap_iter *iter, loff_t pos, |
9060bc4d | 139 | unsigned len); |
40405ddd | 140 | void (*put_folio)(struct inode *inode, loff_t pos, unsigned copied, |
80baab88 | 141 | struct folio *folio); |
d7b64041 DC |
142 | |
143 | /* | |
144 | * Check that the cached iomap still maps correctly to the filesystem's | |
145 | * internal extent map. FS internal extent maps can change while iomap | |
146 | * is iterating a cached iomap, so this hook allows iomap to detect that | |
147 | * the iomap needs to be refreshed during a long running write | |
148 | * operation. | |
149 | * | |
150 | * The filesystem can store internal state (e.g. a sequence number) in | |
151 | * iomap->validity_cookie when the iomap is first mapped to be able to | |
152 | * detect changes between mapping time and whenever .iomap_valid() is | |
153 | * called. | |
154 | * | |
155 | * This is called with the folio over the specified file position held | |
156 | * locked by the iomap code. | |
157 | */ | |
158 | bool (*iomap_valid)(struct inode *inode, const struct iomap *iomap); | |
ae259a9c CH |
159 | }; |
160 | ||
161 | /* | |
162 | * Flags for iomap_begin / iomap_end. No flag implies a read. | |
163 | */ | |
d33fd776 CH |
164 | #define IOMAP_WRITE (1 << 0) /* writing, must allocate blocks */ |
165 | #define IOMAP_ZERO (1 << 1) /* zeroing operation, may skip holes */ | |
166 | #define IOMAP_REPORT (1 << 2) /* report extent status, e.g. FIEMAP */ | |
9484ab1b | 167 | #define IOMAP_FAULT (1 << 3) /* mapping for page fault */ |
ff6a9292 | 168 | #define IOMAP_DIRECT (1 << 4) /* direct I/O */ |
9ecac0ef | 169 | #define IOMAP_NOWAIT (1 << 5) /* do not block */ |
213f6271 | 170 | #define IOMAP_OVERWRITE_ONLY (1 << 6) /* only pure overwrites allowed */ |
b74b1293 | 171 | #define IOMAP_UNSHARE (1 << 7) /* unshare_file_range */ |
952da063 CH |
172 | #ifdef CONFIG_FS_DAX |
173 | #define IOMAP_DAX (1 << 8) /* DAX mapping */ | |
174 | #else | |
175 | #define IOMAP_DAX 0 | |
176 | #endif /* CONFIG_FS_DAX */ | |
ae259a9c CH |
177 | |
178 | struct iomap_ops { | |
179 | /* | |
180 | * Return the existing mapping at pos, or reserve space starting at | |
181 | * pos for up to length, as long as we can do it as a single mapping. | |
182 | * The actual length is returned in iomap->length. | |
183 | */ | |
184 | int (*iomap_begin)(struct inode *inode, loff_t pos, loff_t length, | |
c039b997 GR |
185 | unsigned flags, struct iomap *iomap, |
186 | struct iomap *srcmap); | |
ae259a9c CH |
187 | |
188 | /* | |
189 | * Commit and/or unreserve space previous allocated using iomap_begin. | |
190 | * Written indicates the length of the successful write operation which | |
191 | * needs to be commited, while the rest needs to be unreserved. | |
192 | * Written might be zero if no data was written. | |
193 | */ | |
194 | int (*iomap_end)(struct inode *inode, loff_t pos, loff_t length, | |
195 | ssize_t written, unsigned flags, struct iomap *iomap); | |
199a31c6 CH |
196 | }; |
197 | ||
f4b896c2 CH |
198 | /** |
199 | * struct iomap_iter - Iterate through a range of a file | |
200 | * @inode: Set at the start of the iteration and should not change. | |
201 | * @pos: The current file position we are operating on. It is updated by | |
202 | * calls to iomap_iter(). Treat as read-only in the body. | |
203 | * @len: The remaining length of the file segment we're operating on. | |
204 | * It is updated at the same time as @pos. | |
205 | * @processed: The number of bytes processed by the body in the most recent | |
206 | * iteration, or a negative errno. 0 causes the iteration to stop. | |
207 | * @flags: Zero or more of the iomap_begin flags above. | |
208 | * @iomap: Map describing the I/O iteration | |
209 | * @srcmap: Source map for COW operations | |
210 | */ | |
211 | struct iomap_iter { | |
212 | struct inode *inode; | |
213 | loff_t pos; | |
214 | u64 len; | |
215 | s64 processed; | |
216 | unsigned flags; | |
217 | struct iomap iomap; | |
218 | struct iomap srcmap; | |
786f847f | 219 | void *private; |
f4b896c2 CH |
220 | }; |
221 | ||
222 | int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops); | |
223 | ||
224 | /** | |
225 | * iomap_length - length of the current iomap iteration | |
226 | * @iter: iteration structure | |
227 | * | |
228 | * Returns the length that the operation applies to for the current iteration. | |
229 | */ | |
230 | static inline u64 iomap_length(const struct iomap_iter *iter) | |
231 | { | |
232 | u64 end = iter->iomap.offset + iter->iomap.length; | |
233 | ||
234 | if (iter->srcmap.type != IOMAP_HOLE) | |
235 | end = min(end, iter->srcmap.offset + iter->srcmap.length); | |
236 | return min(iter->len, end - iter->pos); | |
237 | } | |
238 | ||
239 | /** | |
240 | * iomap_iter_srcmap - return the source map for the current iomap iteration | |
241 | * @i: iteration structure | |
242 | * | |
243 | * Write operations on file systems with reflink support might require a | |
244 | * source and a destination map. This function retourns the source map | |
245 | * for a given operation, which may or may no be identical to the destination | |
246 | * map in &i->iomap. | |
247 | */ | |
fad0a1ab | 248 | static inline const struct iomap *iomap_iter_srcmap(const struct iomap_iter *i) |
f4b896c2 CH |
249 | { |
250 | if (i->srcmap.type != IOMAP_HOLE) | |
251 | return &i->srcmap; | |
252 | return &i->iomap; | |
253 | } | |
254 | ||
ae259a9c | 255 | ssize_t iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *from, |
8ff6daa1 | 256 | const struct iomap_ops *ops); |
9c7babf9 DC |
257 | int iomap_file_buffered_write_punch_delalloc(struct inode *inode, |
258 | struct iomap *iomap, loff_t pos, loff_t length, ssize_t written, | |
259 | int (*punch)(struct inode *inode, loff_t pos, loff_t length)); | |
260 | ||
7479c505 | 261 | int iomap_read_folio(struct folio *folio, const struct iomap_ops *ops); |
9d24a13a | 262 | void iomap_readahead(struct readahead_control *, const struct iomap_ops *ops); |
2e7e80f7 | 263 | bool iomap_is_partially_uptodate(struct folio *, size_t from, size_t count); |
98321b51 | 264 | struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos); |
8597447d | 265 | bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags); |
8306a5f5 | 266 | void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len); |
3590c4d8 | 267 | int iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, |
8ff6daa1 | 268 | const struct iomap_ops *ops); |
ae259a9c | 269 | int iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, |
8ff6daa1 | 270 | bool *did_zero, const struct iomap_ops *ops); |
ae259a9c | 271 | int iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, |
8ff6daa1 | 272 | const struct iomap_ops *ops); |
5780a02f SJ |
273 | vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, |
274 | const struct iomap_ops *ops); | |
8be9f564 | 275 | int iomap_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, |
27328818 | 276 | u64 start, u64 len, const struct iomap_ops *ops); |
0ed3b0d4 AG |
277 | loff_t iomap_seek_hole(struct inode *inode, loff_t offset, |
278 | const struct iomap_ops *ops); | |
279 | loff_t iomap_seek_data(struct inode *inode, loff_t offset, | |
280 | const struct iomap_ops *ops); | |
89eb1906 CH |
281 | sector_t iomap_bmap(struct address_space *mapping, sector_t bno, |
282 | const struct iomap_ops *ops); | |
ae259a9c | 283 | |
598ecfba CH |
284 | /* |
285 | * Structure for writeback I/O completions. | |
286 | */ | |
287 | struct iomap_ioend { | |
288 | struct list_head io_list; /* next ioend in chain */ | |
289 | u16 io_type; | |
290 | u16 io_flags; /* IOMAP_F_* */ | |
ebb7fb15 | 291 | u32 io_folios; /* folios added to ioend */ |
598ecfba CH |
292 | struct inode *io_inode; /* file being written to */ |
293 | size_t io_size; /* size of the extent */ | |
294 | loff_t io_offset; /* offset in the file */ | |
ebb7fb15 | 295 | sector_t io_sector; /* start sector of ioend */ |
598ecfba CH |
296 | struct bio *io_bio; /* bio being built */ |
297 | struct bio io_inline_bio; /* MUST BE LAST! */ | |
298 | }; | |
299 | ||
300 | struct iomap_writeback_ops { | |
301 | /* | |
302 | * Required, maps the blocks so that writeback can be performed on | |
303 | * the range starting at offset. | |
304 | */ | |
305 | int (*map_blocks)(struct iomap_writepage_ctx *wpc, struct inode *inode, | |
306 | loff_t offset); | |
307 | ||
308 | /* | |
309 | * Optional, allows the file systems to perform actions just before | |
310 | * submitting the bio and/or override the bio end_io handler for complex | |
311 | * operations like copy on write extent manipulation or unwritten extent | |
312 | * conversions. | |
313 | */ | |
314 | int (*prepare_ioend)(struct iomap_ioend *ioend, int status); | |
315 | ||
316 | /* | |
317 | * Optional, allows the file system to discard state on a page where | |
318 | * we failed to submit any I/O. | |
319 | */ | |
6e478521 | 320 | void (*discard_folio)(struct folio *folio, loff_t pos); |
598ecfba CH |
321 | }; |
322 | ||
323 | struct iomap_writepage_ctx { | |
324 | struct iomap iomap; | |
325 | struct iomap_ioend *ioend; | |
326 | const struct iomap_writeback_ops *ops; | |
327 | }; | |
328 | ||
329 | void iomap_finish_ioends(struct iomap_ioend *ioend, int error); | |
330 | void iomap_ioend_try_merge(struct iomap_ioend *ioend, | |
6e552494 | 331 | struct list_head *more_ioends); |
598ecfba | 332 | void iomap_sort_ioends(struct list_head *ioend_list); |
598ecfba CH |
333 | int iomap_writepages(struct address_space *mapping, |
334 | struct writeback_control *wbc, struct iomap_writepage_ctx *wpc, | |
335 | const struct iomap_writeback_ops *ops); | |
336 | ||
ff6a9292 CH |
337 | /* |
338 | * Flags for direct I/O ->end_io: | |
339 | */ | |
340 | #define IOMAP_DIO_UNWRITTEN (1 << 0) /* covers unwritten extent(s) */ | |
341 | #define IOMAP_DIO_COW (1 << 1) /* covers COW extent(s) */ | |
838c4f3d CH |
342 | |
343 | struct iomap_dio_ops { | |
344 | int (*end_io)(struct kiocb *iocb, ssize_t size, int error, | |
345 | unsigned flags); | |
3e08773c CH |
346 | void (*submit_io)(const struct iomap_iter *iter, struct bio *bio, |
347 | loff_t file_offset); | |
908c5490 CH |
348 | |
349 | /* | |
350 | * Filesystems wishing to attach private information to a direct io bio | |
351 | * must provide a ->submit_io method that attaches the additional | |
352 | * information to the bio and changes the ->bi_end_io callback to a | |
353 | * custom function. This function should, at a minimum, perform any | |
354 | * relevant post-processing of the bio and end with a call to | |
355 | * iomap_dio_bio_end_io. | |
356 | */ | |
357 | struct bio_set *bio_set; | |
838c4f3d CH |
358 | }; |
359 | ||
2f632965 CH |
360 | /* |
361 | * Wait for the I/O to complete in iomap_dio_rw even if the kiocb is not | |
362 | * synchronous. | |
363 | */ | |
364 | #define IOMAP_DIO_FORCE_WAIT (1 << 0) | |
365 | ||
213f6271 CH |
366 | /* |
367 | * Do not allocate blocks or zero partial blocks, but instead fall back to | |
368 | * the caller by returning -EAGAIN. Used to optimize direct I/O writes that | |
369 | * are not aligned to the file system block size. | |
370 | */ | |
371 | #define IOMAP_DIO_OVERWRITE_ONLY (1 << 1) | |
372 | ||
97308f8b AG |
373 | /* |
374 | * When a page fault occurs, return a partial synchronous result and allow | |
375 | * the caller to retry the rest of the operation after dealing with the page | |
376 | * fault. | |
377 | */ | |
378 | #define IOMAP_DIO_PARTIAL (1 << 2) | |
379 | ||
ff6a9292 | 380 | ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, |
13ef9544 | 381 | const struct iomap_ops *ops, const struct iomap_dio_ops *dops, |
786f847f | 382 | unsigned int dio_flags, void *private, size_t done_before); |
c3d4ed1a CH |
383 | struct iomap_dio *__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, |
384 | const struct iomap_ops *ops, const struct iomap_dio_ops *dops, | |
786f847f | 385 | unsigned int dio_flags, void *private, size_t done_before); |
c3d4ed1a | 386 | ssize_t iomap_dio_complete(struct iomap_dio *dio); |
908c5490 | 387 | void iomap_dio_bio_end_io(struct bio *bio); |
ff6a9292 | 388 | |
67482129 DW |
389 | #ifdef CONFIG_SWAP |
390 | struct file; | |
391 | struct swap_info_struct; | |
392 | ||
393 | int iomap_swapfile_activate(struct swap_info_struct *sis, | |
394 | struct file *swap_file, sector_t *pagespan, | |
395 | const struct iomap_ops *ops); | |
396 | #else | |
397 | # define iomap_swapfile_activate(sis, swapfile, pagespan, ops) (-EIO) | |
398 | #endif /* CONFIG_SWAP */ | |
399 | ||
199a31c6 | 400 | #endif /* LINUX_IOMAP_H */ |