Commit | Line | Data |
---|---|---|
0b61f8a4 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 | 2 | /* |
7b718769 | 3 | * Copyright (c) 2000-2005 Silicon Graphics, Inc. |
98c1a7c0 | 4 | * Copyright (c) 2016-2018 Christoph Hellwig. |
7b718769 | 5 | * All Rights Reserved. |
1da177e4 | 6 | */ |
1da177e4 | 7 | #include "xfs.h" |
70a9883c | 8 | #include "xfs_shared.h" |
239880ef DC |
9 | #include "xfs_format.h" |
10 | #include "xfs_log_format.h" | |
11 | #include "xfs_trans_resv.h" | |
1da177e4 | 12 | #include "xfs_mount.h" |
1da177e4 | 13 | #include "xfs_inode.h" |
239880ef | 14 | #include "xfs_trans.h" |
1da177e4 | 15 | #include "xfs_iomap.h" |
0b1b213f | 16 | #include "xfs_trace.h" |
3ed3a434 | 17 | #include "xfs_bmap.h" |
68988114 | 18 | #include "xfs_bmap_util.h" |
ef473667 | 19 | #include "xfs_reflink.h" |
c2beff99 DW |
20 | #include "xfs_errortag.h" |
21 | #include "xfs_error.h" | |
1da177e4 | 22 | |
fbcc0256 | 23 | struct xfs_writepage_ctx { |
598ecfba | 24 | struct iomap_writepage_ctx ctx; |
d9252d52 | 25 | unsigned int data_seq; |
e666aa37 | 26 | unsigned int cow_seq; |
fbcc0256 DC |
27 | }; |
28 | ||
598ecfba CH |
29 | static inline struct xfs_writepage_ctx * |
30 | XFS_WPC(struct iomap_writepage_ctx *ctx) | |
31 | { | |
32 | return container_of(ctx, struct xfs_writepage_ctx, ctx); | |
33 | } | |
34 | ||
fc0063c4 CH |
35 | /* |
36 | * Fast and loose check if this write could update the on-disk inode size. | |
37 | */ | |
598ecfba | 38 | static inline bool xfs_ioend_is_append(struct iomap_ioend *ioend) |
fc0063c4 CH |
39 | { |
40 | return ioend->io_offset + ioend->io_size > | |
13d2c10b | 41 | XFS_I(ioend->io_inode)->i_disk_size; |
fc0063c4 CH |
42 | } |
43 | ||
ba87ea69 | 44 | /* |
2813d682 | 45 | * Update on-disk file size now that data has been written to disk. |
ba87ea69 | 46 | */ |
e7a3d7e7 BF |
47 | int |
48 | xfs_setfilesize( | |
2ba66237 | 49 | struct xfs_inode *ip, |
2ba66237 CH |
50 | xfs_off_t offset, |
51 | size_t size) | |
ba87ea69 | 52 | { |
e7a3d7e7 BF |
53 | struct xfs_mount *mp = ip->i_mount; |
54 | struct xfs_trans *tp; | |
ba87ea69 | 55 | xfs_fsize_t isize; |
e7a3d7e7 BF |
56 | int error; |
57 | ||
58 | error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp); | |
59 | if (error) | |
60 | return error; | |
ba87ea69 | 61 | |
aa6bf01d | 62 | xfs_ilock(ip, XFS_ILOCK_EXCL); |
2ba66237 | 63 | isize = xfs_new_eof(ip, offset + size); |
281627df CH |
64 | if (!isize) { |
65 | xfs_iunlock(ip, XFS_ILOCK_EXCL); | |
4906e215 | 66 | xfs_trans_cancel(tp); |
281627df | 67 | return 0; |
ba87ea69 LM |
68 | } |
69 | ||
2ba66237 | 70 | trace_xfs_setfilesize(ip, offset, size); |
281627df | 71 | |
13d2c10b | 72 | ip->i_disk_size = isize; |
281627df CH |
73 | xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); |
74 | xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); | |
75 | ||
70393313 | 76 | return xfs_trans_commit(tp); |
77d7a0c2 DC |
77 | } |
78 | ||
0829c360 | 79 | /* |
5ec4fabb | 80 | * IO write completion. |
f6d6d4fc CH |
81 | */ |
82 | STATIC void | |
cb357bf3 | 83 | xfs_end_ioend( |
598ecfba | 84 | struct iomap_ioend *ioend) |
0829c360 | 85 | { |
0e51a8e1 | 86 | struct xfs_inode *ip = XFS_I(ioend->io_inode); |
5ca5916b | 87 | struct xfs_mount *mp = ip->i_mount; |
787eb485 CH |
88 | xfs_off_t offset = ioend->io_offset; |
89 | size_t size = ioend->io_size; | |
73d30d48 | 90 | unsigned int nofs_flag; |
4e4cbee9 | 91 | int error; |
ba87ea69 | 92 | |
73d30d48 CH |
93 | /* |
94 | * We can allocate memory here while doing writeback on behalf of | |
95 | * memory reclaim. To avoid memory allocation deadlocks set the | |
96 | * task-wide nofs context for the following operations. | |
97 | */ | |
98 | nofs_flag = memalloc_nofs_save(); | |
99 | ||
af055e37 | 100 | /* |
f9dd7ba4 | 101 | * Just clean up the in-memory structures if the fs has been shut down. |
af055e37 | 102 | */ |
5ca5916b | 103 | if (xfs_is_shutdown(mp)) { |
0e51a8e1 | 104 | error = -EIO; |
787eb485 CH |
105 | goto done; |
106 | } | |
04f658ee | 107 | |
43caeb18 | 108 | /* |
5ca5916b BF |
109 | * Clean up all COW blocks and underlying data fork delalloc blocks on |
110 | * I/O error. The delalloc punch is required because this ioend was | |
111 | * mapped to blocks in the COW fork and the associated pages are no | |
112 | * longer dirty. If we don't remove delalloc blocks here, they become | |
113 | * stale and can corrupt free space accounting on unmount. | |
43caeb18 | 114 | */ |
4e4cbee9 | 115 | error = blk_status_to_errno(ioend->io_bio->bi_status); |
787eb485 | 116 | if (unlikely(error)) { |
5ca5916b | 117 | if (ioend->io_flags & IOMAP_F_SHARED) { |
787eb485 | 118 | xfs_reflink_cancel_cow_range(ip, offset, size, true); |
7348b322 DC |
119 | xfs_bmap_punch_delalloc_range(ip, offset, |
120 | offset + size); | |
5ca5916b | 121 | } |
787eb485 | 122 | goto done; |
43caeb18 DW |
123 | } |
124 | ||
5ec4fabb | 125 | /* |
be225fec | 126 | * Success: commit the COW or unwritten blocks if needed. |
5ec4fabb | 127 | */ |
760fea8b | 128 | if (ioend->io_flags & IOMAP_F_SHARED) |
787eb485 | 129 | error = xfs_reflink_end_cow(ip, offset, size); |
4e087a3b | 130 | else if (ioend->io_type == IOMAP_UNWRITTEN) |
ee70daab | 131 | error = xfs_iomap_write_unwritten(ip, offset, size, false); |
ba87ea69 | 132 | |
7cd3099f BF |
133 | if (!error && xfs_ioend_is_append(ioend)) |
134 | error = xfs_setfilesize(ip, ioend->io_offset, ioend->io_size); | |
04f658ee | 135 | done: |
598ecfba | 136 | iomap_finish_ioends(ioend, error); |
73d30d48 | 137 | memalloc_nofs_restore(nofs_flag); |
3994fc48 DW |
138 | } |
139 | ||
ebb7fb15 DC |
140 | /* |
141 | * Finish all pending IO completions that require transactional modifications. | |
142 | * | |
143 | * We try to merge physical and logically contiguous ioends before completion to | |
144 | * minimise the number of transactions we need to perform during IO completion. | |
145 | * Both unwritten extent conversion and COW remapping need to iterate and modify | |
146 | * one physical extent at a time, so we gain nothing by merging physically | |
147 | * discontiguous extents here. | |
148 | * | |
149 | * The ioend chain length that we can be processing here is largely unbound in | |
150 | * length and we may have to perform significant amounts of work on each ioend | |
151 | * to complete it. Hence we have to be careful about holding the CPU for too | |
152 | * long in this loop. | |
153 | */ | |
cb357bf3 DW |
154 | void |
155 | xfs_end_io( | |
156 | struct work_struct *work) | |
157 | { | |
433dad94 CH |
158 | struct xfs_inode *ip = |
159 | container_of(work, struct xfs_inode, i_ioend_work); | |
598ecfba | 160 | struct iomap_ioend *ioend; |
433dad94 | 161 | struct list_head tmp; |
cb357bf3 DW |
162 | unsigned long flags; |
163 | ||
cb357bf3 | 164 | spin_lock_irqsave(&ip->i_ioend_lock, flags); |
433dad94 | 165 | list_replace_init(&ip->i_ioend_list, &tmp); |
cb357bf3 DW |
166 | spin_unlock_irqrestore(&ip->i_ioend_lock, flags); |
167 | ||
598ecfba CH |
168 | iomap_sort_ioends(&tmp); |
169 | while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend, | |
433dad94 | 170 | io_list))) { |
cb357bf3 | 171 | list_del_init(&ioend->io_list); |
6e552494 | 172 | iomap_ioend_try_merge(ioend, &tmp); |
cb357bf3 | 173 | xfs_end_ioend(ioend); |
ebb7fb15 | 174 | cond_resched(); |
cb357bf3 DW |
175 | } |
176 | } | |
177 | ||
0e51a8e1 CH |
178 | STATIC void |
179 | xfs_end_bio( | |
180 | struct bio *bio) | |
0829c360 | 181 | { |
598ecfba | 182 | struct iomap_ioend *ioend = bio->bi_private; |
cb357bf3 | 183 | struct xfs_inode *ip = XFS_I(ioend->io_inode); |
cb357bf3 | 184 | unsigned long flags; |
0829c360 | 185 | |
598ecfba CH |
186 | spin_lock_irqsave(&ip->i_ioend_lock, flags); |
187 | if (list_empty(&ip->i_ioend_list)) | |
188 | WARN_ON_ONCE(!queue_work(ip->i_mount->m_unwritten_workqueue, | |
189 | &ip->i_ioend_work)); | |
190 | list_add_tail(&ioend->io_list, &ip->i_ioend_list); | |
191 | spin_unlock_irqrestore(&ip->i_ioend_lock, flags); | |
0829c360 CH |
192 | } |
193 | ||
d9252d52 BF |
194 | /* |
195 | * Fast revalidation of the cached writeback mapping. Return true if the current | |
196 | * mapping is valid, false otherwise. | |
197 | */ | |
198 | static bool | |
199 | xfs_imap_valid( | |
598ecfba | 200 | struct iomap_writepage_ctx *wpc, |
d9252d52 | 201 | struct xfs_inode *ip, |
4e087a3b | 202 | loff_t offset) |
d9252d52 | 203 | { |
4e087a3b CH |
204 | if (offset < wpc->iomap.offset || |
205 | offset >= wpc->iomap.offset + wpc->iomap.length) | |
d9252d52 BF |
206 | return false; |
207 | /* | |
208 | * If this is a COW mapping, it is sufficient to check that the mapping | |
209 | * covers the offset. Be careful to check this first because the caller | |
210 | * can revalidate a COW mapping without updating the data seqno. | |
211 | */ | |
760fea8b | 212 | if (wpc->iomap.flags & IOMAP_F_SHARED) |
d9252d52 BF |
213 | return true; |
214 | ||
215 | /* | |
216 | * This is not a COW mapping. Check the sequence number of the data fork | |
217 | * because concurrent changes could have invalidated the extent. Check | |
218 | * the COW fork because concurrent changes since the last time we | |
219 | * checked (and found nothing at this offset) could have added | |
220 | * overlapping blocks. | |
221 | */ | |
c2beff99 DW |
222 | if (XFS_WPC(wpc)->data_seq != READ_ONCE(ip->i_df.if_seq)) { |
223 | trace_xfs_wb_data_iomap_invalid(ip, &wpc->iomap, | |
224 | XFS_WPC(wpc)->data_seq, XFS_DATA_FORK); | |
d9252d52 | 225 | return false; |
c2beff99 | 226 | } |
d9252d52 | 227 | if (xfs_inode_has_cow_data(ip) && |
c2beff99 DW |
228 | XFS_WPC(wpc)->cow_seq != READ_ONCE(ip->i_cowfp->if_seq)) { |
229 | trace_xfs_wb_cow_iomap_invalid(ip, &wpc->iomap, | |
230 | XFS_WPC(wpc)->cow_seq, XFS_COW_FORK); | |
d9252d52 | 231 | return false; |
c2beff99 | 232 | } |
d9252d52 BF |
233 | return true; |
234 | } | |
235 | ||
4ad765ed CH |
236 | /* |
237 | * Pass in a dellalloc extent and convert it to real extents, return the real | |
4e087a3b | 238 | * extent that maps offset_fsb in wpc->iomap. |
4ad765ed CH |
239 | * |
240 | * The current page is held locked so nothing could have removed the block | |
7588cbee CH |
241 | * backing offset_fsb, although it could have moved from the COW to the data |
242 | * fork by another thread. | |
4ad765ed CH |
243 | */ |
244 | static int | |
245 | xfs_convert_blocks( | |
598ecfba | 246 | struct iomap_writepage_ctx *wpc, |
4ad765ed | 247 | struct xfs_inode *ip, |
760fea8b | 248 | int whichfork, |
4e087a3b | 249 | loff_t offset) |
4ad765ed CH |
250 | { |
251 | int error; | |
598ecfba CH |
252 | unsigned *seq; |
253 | ||
254 | if (whichfork == XFS_COW_FORK) | |
255 | seq = &XFS_WPC(wpc)->cow_seq; | |
256 | else | |
257 | seq = &XFS_WPC(wpc)->data_seq; | |
4ad765ed CH |
258 | |
259 | /* | |
4e087a3b CH |
260 | * Attempt to allocate whatever delalloc extent currently backs offset |
261 | * and put the result into wpc->iomap. Allocate in a loop because it | |
262 | * may take several attempts to allocate real blocks for a contiguous | |
263 | * delalloc extent if free space is sufficiently fragmented. | |
4ad765ed CH |
264 | */ |
265 | do { | |
760fea8b | 266 | error = xfs_bmapi_convert_delalloc(ip, whichfork, offset, |
598ecfba | 267 | &wpc->iomap, seq); |
4ad765ed CH |
268 | if (error) |
269 | return error; | |
4e087a3b | 270 | } while (wpc->iomap.offset + wpc->iomap.length <= offset); |
4ad765ed CH |
271 | |
272 | return 0; | |
273 | } | |
274 | ||
598ecfba | 275 | static int |
1da177e4 | 276 | xfs_map_blocks( |
598ecfba | 277 | struct iomap_writepage_ctx *wpc, |
1da177e4 | 278 | struct inode *inode, |
5c665e5b | 279 | loff_t offset) |
1da177e4 | 280 | { |
a206c817 CH |
281 | struct xfs_inode *ip = XFS_I(inode); |
282 | struct xfs_mount *mp = ip->i_mount; | |
93407472 | 283 | ssize_t count = i_blocksize(inode); |
b4e29032 CH |
284 | xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); |
285 | xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count); | |
c2f09217 DW |
286 | xfs_fileoff_t cow_fsb; |
287 | int whichfork; | |
5c665e5b | 288 | struct xfs_bmbt_irec imap; |
060d4eaa | 289 | struct xfs_iext_cursor icur; |
7588cbee | 290 | int retries = 0; |
a206c817 | 291 | int error = 0; |
a206c817 | 292 | |
75c8c50f | 293 | if (xfs_is_shutdown(mp)) |
d9252d52 BF |
294 | return -EIO; |
295 | ||
c2beff99 DW |
296 | XFS_ERRORTAG_DELAY(mp, XFS_ERRTAG_WB_DELAY_MS); |
297 | ||
889c65b3 CH |
298 | /* |
299 | * COW fork blocks can overlap data fork blocks even if the blocks | |
300 | * aren't shared. COW I/O always takes precedent, so we must always | |
301 | * check for overlap on reflink inodes unless the mapping is already a | |
e666aa37 CH |
302 | * COW one, or the COW fork hasn't changed from the last time we looked |
303 | * at it. | |
304 | * | |
305 | * It's safe to check the COW fork if_seq here without the ILOCK because | |
306 | * we've indirectly protected against concurrent updates: writeback has | |
307 | * the page locked, which prevents concurrent invalidations by reflink | |
308 | * and directio and prevents concurrent buffered writes to the same | |
309 | * page. Changes to if_seq always happen under i_lock, which protects | |
310 | * against concurrent updates and provides a memory barrier on the way | |
311 | * out that ensures that we always see the current value. | |
889c65b3 | 312 | */ |
4e087a3b | 313 | if (xfs_imap_valid(wpc, ip, offset)) |
889c65b3 CH |
314 | return 0; |
315 | ||
889c65b3 CH |
316 | /* |
317 | * If we don't have a valid map, now it's time to get a new one for this | |
318 | * offset. This will convert delayed allocations (including COW ones) | |
319 | * into real extents. If we return without a valid map, it means we | |
320 | * landed in a hole and we skip the block. | |
321 | */ | |
7588cbee | 322 | retry: |
c2f09217 DW |
323 | cow_fsb = NULLFILEOFF; |
324 | whichfork = XFS_DATA_FORK; | |
988ef927 | 325 | xfs_ilock(ip, XFS_ILOCK_SHARED); |
b2197a36 | 326 | ASSERT(!xfs_need_iread_extents(&ip->i_df)); |
060d4eaa CH |
327 | |
328 | /* | |
329 | * Check if this is offset is covered by a COW extents, and if yes use | |
330 | * it directly instead of looking up anything in the data fork. | |
331 | */ | |
51d62690 | 332 | if (xfs_inode_has_cow_data(ip) && |
e666aa37 CH |
333 | xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap)) |
334 | cow_fsb = imap.br_startoff; | |
335 | if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) { | |
598ecfba | 336 | XFS_WPC(wpc)->cow_seq = READ_ONCE(ip->i_cowfp->if_seq); |
5c665e5b | 337 | xfs_iunlock(ip, XFS_ILOCK_SHARED); |
be225fec | 338 | |
760fea8b | 339 | whichfork = XFS_COW_FORK; |
5c665e5b CH |
340 | goto allocate_blocks; |
341 | } | |
342 | ||
343 | /* | |
d9252d52 BF |
344 | * No COW extent overlap. Revalidate now that we may have updated |
345 | * ->cow_seq. If the data mapping is still valid, we're done. | |
5c665e5b | 346 | */ |
4e087a3b | 347 | if (xfs_imap_valid(wpc, ip, offset)) { |
5c665e5b CH |
348 | xfs_iunlock(ip, XFS_ILOCK_SHARED); |
349 | return 0; | |
350 | } | |
351 | ||
352 | /* | |
353 | * If we don't have a valid map, now it's time to get a new one for this | |
354 | * offset. This will convert delayed allocations (including COW ones) | |
355 | * into real extents. | |
356 | */ | |
3345746e CH |
357 | if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap)) |
358 | imap.br_startoff = end_fsb; /* fake a hole past EOF */ | |
598ecfba | 359 | XFS_WPC(wpc)->data_seq = READ_ONCE(ip->i_df.if_seq); |
8ff2957d | 360 | xfs_iunlock(ip, XFS_ILOCK_SHARED); |
a206c817 | 361 | |
12df89f2 | 362 | /* landed in a hole or beyond EOF? */ |
3345746e | 363 | if (imap.br_startoff > offset_fsb) { |
3345746e | 364 | imap.br_blockcount = imap.br_startoff - offset_fsb; |
5c665e5b | 365 | imap.br_startoff = offset_fsb; |
5c665e5b | 366 | imap.br_startblock = HOLESTARTBLOCK; |
be225fec | 367 | imap.br_state = XFS_EXT_NORM; |
8ff2957d | 368 | } |
e2f6ad46 | 369 | |
12df89f2 CH |
370 | /* |
371 | * Truncate to the next COW extent if there is one. This is the only | |
372 | * opportunity to do this because we can skip COW fork lookups for the | |
373 | * subsequent blocks in the mapping; however, the requirement to treat | |
374 | * the COW range separately remains. | |
375 | */ | |
376 | if (cow_fsb != NULLFILEOFF && | |
377 | cow_fsb < imap.br_startoff + imap.br_blockcount) | |
378 | imap.br_blockcount = cow_fsb - imap.br_startoff; | |
379 | ||
380 | /* got a delalloc extent? */ | |
381 | if (imap.br_startblock != HOLESTARTBLOCK && | |
382 | isnullstartblock(imap.br_startblock)) | |
383 | goto allocate_blocks; | |
384 | ||
304a68b9 | 385 | xfs_bmbt_to_iomap(ip, &wpc->iomap, &imap, 0, 0, XFS_WPC(wpc)->data_seq); |
760fea8b | 386 | trace_xfs_map_blocks_found(ip, offset, count, whichfork, &imap); |
5c665e5b CH |
387 | return 0; |
388 | allocate_blocks: | |
760fea8b | 389 | error = xfs_convert_blocks(wpc, ip, whichfork, offset); |
7588cbee CH |
390 | if (error) { |
391 | /* | |
392 | * If we failed to find the extent in the COW fork we might have | |
393 | * raced with a COW to data fork conversion or truncate. | |
394 | * Restart the lookup to catch the extent in the data fork for | |
395 | * the former case, but prevent additional retries to avoid | |
396 | * looping forever for the latter case. | |
397 | */ | |
760fea8b | 398 | if (error == -EAGAIN && whichfork == XFS_COW_FORK && !retries++) |
7588cbee CH |
399 | goto retry; |
400 | ASSERT(error != -EAGAIN); | |
5c665e5b | 401 | return error; |
7588cbee | 402 | } |
4ad765ed CH |
403 | |
404 | /* | |
405 | * Due to merging the return real extent might be larger than the | |
406 | * original delalloc one. Trim the return extent to the next COW | |
407 | * boundary again to force a re-lookup. | |
408 | */ | |
760fea8b | 409 | if (whichfork != XFS_COW_FORK && cow_fsb != NULLFILEOFF) { |
4e087a3b CH |
410 | loff_t cow_offset = XFS_FSB_TO_B(mp, cow_fsb); |
411 | ||
412 | if (cow_offset < wpc->iomap.offset + wpc->iomap.length) | |
413 | wpc->iomap.length = cow_offset - wpc->iomap.offset; | |
414 | } | |
4ad765ed | 415 | |
4e087a3b CH |
416 | ASSERT(wpc->iomap.offset <= offset); |
417 | ASSERT(wpc->iomap.offset + wpc->iomap.length > offset); | |
760fea8b | 418 | trace_xfs_map_blocks_alloc(ip, offset, count, whichfork, &imap); |
8ff2957d | 419 | return 0; |
1da177e4 LT |
420 | } |
421 | ||
598ecfba CH |
422 | static int |
423 | xfs_prepare_ioend( | |
424 | struct iomap_ioend *ioend, | |
e10de372 | 425 | int status) |
f6d6d4fc | 426 | { |
73d30d48 CH |
427 | unsigned int nofs_flag; |
428 | ||
429 | /* | |
430 | * We can allocate memory here while doing writeback on behalf of | |
431 | * memory reclaim. To avoid memory allocation deadlocks set the | |
432 | * task-wide nofs context for the following operations. | |
433 | */ | |
434 | nofs_flag = memalloc_nofs_save(); | |
435 | ||
5eda4300 | 436 | /* Convert CoW extents to regular */ |
760fea8b | 437 | if (!status && (ioend->io_flags & IOMAP_F_SHARED)) { |
5eda4300 DW |
438 | status = xfs_reflink_convert_cow(XFS_I(ioend->io_inode), |
439 | ioend->io_offset, ioend->io_size); | |
440 | } | |
441 | ||
73d30d48 CH |
442 | memalloc_nofs_restore(nofs_flag); |
443 | ||
7adb8f14 BF |
444 | /* send ioends that might require a transaction to the completion wq */ |
445 | if (xfs_ioend_is_append(ioend) || ioend->io_type == IOMAP_UNWRITTEN || | |
446 | (ioend->io_flags & IOMAP_F_SHARED)) | |
598ecfba CH |
447 | ioend->io_bio->bi_end_io = xfs_end_bio; |
448 | return status; | |
f6d6d4fc CH |
449 | } |
450 | ||
3ed3a434 | 451 | /* |
8ac5b996 DC |
452 | * If the folio has delalloc blocks on it, the caller is asking us to punch them |
453 | * out. If we don't, we can leave a stale delalloc mapping covered by a clean | |
454 | * page that needs to be dirtied again before the delalloc mapping can be | |
455 | * converted. This stale delalloc mapping can trip up a later direct I/O read | |
456 | * operation on the same region. | |
3ed3a434 | 457 | * |
8ac5b996 | 458 | * We prevent this by truncating away the delalloc regions on the folio. Because |
82cb1417 CH |
459 | * they are delalloc, we can do this without needing a transaction. Indeed - if |
460 | * we get ENOSPC errors, we have to be able to do this truncation without a | |
8ac5b996 DC |
461 | * transaction as there is no space left for block reservation (typically why |
462 | * we see a ENOSPC in writeback). | |
3ed3a434 | 463 | */ |
598ecfba | 464 | static void |
6e478521 MWO |
465 | xfs_discard_folio( |
466 | struct folio *folio, | |
467 | loff_t pos) | |
3ed3a434 | 468 | { |
7348b322 | 469 | struct xfs_inode *ip = XFS_I(folio->mapping->host); |
03625721 | 470 | struct xfs_mount *mp = ip->i_mount; |
03625721 | 471 | int error; |
3ed3a434 | 472 | |
75c8c50f | 473 | if (xfs_is_shutdown(mp)) |
e9c3a8e8 | 474 | return; |
e8c3753c | 475 | |
4ab45e25 | 476 | xfs_alert_ratelimited(mp, |
6e478521 MWO |
477 | "page discard on page "PTR_FMT", inode 0x%llx, pos %llu.", |
478 | folio, ip->i_ino, pos); | |
3ed3a434 | 479 | |
8ac5b996 DC |
480 | /* |
481 | * The end of the punch range is always the offset of the the first | |
482 | * byte of the next folio. Hence the end offset is only dependent on the | |
483 | * folio itself and not the start offset that is passed in. | |
484 | */ | |
7348b322 | 485 | error = xfs_bmap_punch_delalloc_range(ip, pos, |
8ac5b996 | 486 | folio_pos(folio) + folio_size(folio)); |
7348b322 | 487 | |
75c8c50f | 488 | if (error && !xfs_is_shutdown(mp)) |
03625721 | 489 | xfs_alert(mp, "page discard unable to remove delalloc mapping."); |
3ed3a434 DC |
490 | } |
491 | ||
598ecfba CH |
492 | static const struct iomap_writeback_ops xfs_writeback_ops = { |
493 | .map_blocks = xfs_map_blocks, | |
494 | .prepare_ioend = xfs_prepare_ioend, | |
6e478521 | 495 | .discard_folio = xfs_discard_folio, |
598ecfba | 496 | }; |
f51623b2 | 497 | |
7d4fb40a NS |
498 | STATIC int |
499 | xfs_vm_writepages( | |
500 | struct address_space *mapping, | |
501 | struct writeback_control *wbc) | |
502 | { | |
be225fec | 503 | struct xfs_writepage_ctx wpc = { }; |
fbcc0256 | 504 | |
756b1c34 DC |
505 | /* |
506 | * Writing back data in a transaction context can result in recursive | |
507 | * transactions. This is bad, so issue a warning and get out of here. | |
508 | */ | |
509 | if (WARN_ON_ONCE(current->journal_info)) | |
510 | return 0; | |
511 | ||
b3aea4ed | 512 | xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED); |
598ecfba | 513 | return iomap_writepages(mapping, wbc, &wpc.ctx, &xfs_writeback_ops); |
7d4fb40a NS |
514 | } |
515 | ||
6e2608df DW |
516 | STATIC int |
517 | xfs_dax_writepages( | |
518 | struct address_space *mapping, | |
519 | struct writeback_control *wbc) | |
520 | { | |
30fa529e CH |
521 | struct xfs_inode *ip = XFS_I(mapping->host); |
522 | ||
523 | xfs_iflags_clear(ip, XFS_ITRUNCATED); | |
6e2608df | 524 | return dax_writeback_mapping_range(mapping, |
3f666c56 | 525 | xfs_inode_buftarg(ip)->bt_daxdev, wbc); |
6e2608df DW |
526 | } |
527 | ||
1da177e4 | 528 | STATIC sector_t |
e4c573bb | 529 | xfs_vm_bmap( |
1da177e4 LT |
530 | struct address_space *mapping, |
531 | sector_t block) | |
532 | { | |
b84e7722 | 533 | struct xfs_inode *ip = XFS_I(mapping->host); |
1da177e4 | 534 | |
b84e7722 | 535 | trace_xfs_vm_bmap(ip); |
db1327b1 DW |
536 | |
537 | /* | |
538 | * The swap code (ab-)uses ->bmap to get a block mapping and then | |
793057e1 | 539 | * bypasses the file system for actual I/O. We really can't allow |
db1327b1 | 540 | * that on reflinks inodes, so we have to skip out here. And yes, |
eb5e248d DW |
541 | * 0 is the magic code for a bmap error. |
542 | * | |
543 | * Since we don't pass back blockdev info, we can't return bmap | |
544 | * information for rt files either. | |
db1327b1 | 545 | */ |
66ae56a5 | 546 | if (xfs_is_cow_inode(ip) || XFS_IS_REALTIME_INODE(ip)) |
db1327b1 | 547 | return 0; |
690c2a38 | 548 | return iomap_bmap(mapping, block, &xfs_read_iomap_ops); |
1da177e4 LT |
549 | } |
550 | ||
551 | STATIC int | |
7479c505 | 552 | xfs_vm_read_folio( |
1da177e4 | 553 | struct file *unused, |
7479c505 | 554 | struct folio *folio) |
1da177e4 | 555 | { |
7479c505 | 556 | return iomap_read_folio(folio, &xfs_read_iomap_ops); |
1da177e4 LT |
557 | } |
558 | ||
9d24a13a MWO |
559 | STATIC void |
560 | xfs_vm_readahead( | |
561 | struct readahead_control *rac) | |
1da177e4 | 562 | { |
9d24a13a | 563 | iomap_readahead(rac, &xfs_read_iomap_ops); |
22e757a4 DC |
564 | } |
565 | ||
67482129 DW |
566 | static int |
567 | xfs_iomap_swapfile_activate( | |
568 | struct swap_info_struct *sis, | |
569 | struct file *swap_file, | |
570 | sector_t *span) | |
571 | { | |
30fa529e | 572 | sis->bdev = xfs_inode_buftarg(XFS_I(file_inode(swap_file)))->bt_bdev; |
690c2a38 CH |
573 | return iomap_swapfile_activate(sis, swap_file, span, |
574 | &xfs_read_iomap_ops); | |
67482129 DW |
575 | } |
576 | ||
f5e54d6e | 577 | const struct address_space_operations xfs_address_space_operations = { |
7479c505 | 578 | .read_folio = xfs_vm_read_folio, |
9d24a13a | 579 | .readahead = xfs_vm_readahead, |
7d4fb40a | 580 | .writepages = xfs_vm_writepages, |
187c82cb | 581 | .dirty_folio = filemap_dirty_folio, |
8597447d | 582 | .release_folio = iomap_release_folio, |
d82354f6 | 583 | .invalidate_folio = iomap_invalidate_folio, |
e4c573bb | 584 | .bmap = xfs_vm_bmap, |
6e2608df | 585 | .direct_IO = noop_direct_IO, |
2ec810d5 | 586 | .migrate_folio = filemap_migrate_folio, |
82cb1417 | 587 | .is_partially_uptodate = iomap_is_partially_uptodate, |
aa261f54 | 588 | .error_remove_page = generic_error_remove_page, |
67482129 | 589 | .swap_activate = xfs_iomap_swapfile_activate, |
1da177e4 | 590 | }; |
6e2608df DW |
591 | |
592 | const struct address_space_operations xfs_dax_aops = { | |
593 | .writepages = xfs_dax_writepages, | |
594 | .direct_IO = noop_direct_IO, | |
46de8b97 | 595 | .dirty_folio = noop_dirty_folio, |
67482129 | 596 | .swap_activate = xfs_iomap_swapfile_activate, |
6e2608df | 597 | }; |