1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
4 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
7 #include <linux/spinlock.h>
8 #include <linux/completion.h>
9 #include <linux/buffer_head.h>
10 #include <linux/blkdev.h>
11 #include <linux/gfs2_ondisk.h>
12 #include <linux/crc32.h>
13 #include <linux/iomap.h>
14 #include <linux/ktime.h>
30 #include "trace_gfs2.h"
32 /* This doesn't need to be that large as max 64 bit pointers in a 4k
33 * block is 512, so __u16 is fine for that. It saves stack space to
37 struct buffer_head *mp_bh[GFS2_MAX_META_HEIGHT];
38 __u16 mp_list[GFS2_MAX_META_HEIGHT];
39 int mp_fheight; /* find_metapath height */
40 int mp_aheight; /* actual height (lookup height) */
43 static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length);
46 * gfs2_unstuffer_folio - unstuff a stuffed inode into a block cached by a folio
48 * @dibh: the dinode buffer
49 * @block: the block number that was allocated
54 static int gfs2_unstuffer_folio(struct gfs2_inode *ip, struct buffer_head *dibh,
55 u64 block, struct folio *folio)
57 struct inode *inode = &ip->i_inode;
59 if (!folio_test_uptodate(folio)) {
60 void *kaddr = kmap_local_folio(folio, 0);
61 u64 dsize = i_size_read(inode);
63 memcpy(kaddr, dibh->b_data + sizeof(struct gfs2_dinode), dsize);
64 memset(kaddr + dsize, 0, folio_size(folio) - dsize);
67 folio_mark_uptodate(folio);
70 if (gfs2_is_jdata(ip)) {
71 struct buffer_head *bh = folio_buffers(folio);
74 bh = create_empty_buffers(folio,
75 BIT(inode->i_blkbits), BIT(BH_Uptodate));
77 if (!buffer_mapped(bh))
78 map_bh(bh, inode->i_sb, block);
80 set_buffer_uptodate(bh);
81 gfs2_trans_add_data(ip->i_gl, bh);
83 folio_mark_dirty(folio);
84 gfs2_ordered_add_inode(ip);
90 static int __gfs2_unstuff_inode(struct gfs2_inode *ip, struct folio *folio)
92 struct buffer_head *bh, *dibh;
93 struct gfs2_dinode *di;
95 int isdir = gfs2_is_dir(ip);
98 error = gfs2_meta_inode_buffer(ip, &dibh);
102 if (i_size_read(&ip->i_inode)) {
103 /* Get a free block, fill it with the stuffed data,
104 and write it out to disk */
107 error = gfs2_alloc_blocks(ip, &block, &n, 0);
111 gfs2_trans_remove_revoke(GFS2_SB(&ip->i_inode), block, 1);
112 error = gfs2_dir_get_new_buffer(ip, block, &bh);
115 gfs2_buffer_copy_tail(bh, sizeof(struct gfs2_meta_header),
116 dibh, sizeof(struct gfs2_dinode));
119 error = gfs2_unstuffer_folio(ip, dibh, block, folio);
125 /* Set up the pointer to the new block */
127 gfs2_trans_add_meta(ip->i_gl, dibh);
128 di = (struct gfs2_dinode *)dibh->b_data;
129 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
131 if (i_size_read(&ip->i_inode)) {
132 *(__be64 *)(di + 1) = cpu_to_be64(block);
133 gfs2_add_inode_blocks(&ip->i_inode, 1);
134 di->di_blocks = cpu_to_be64(gfs2_get_inode_blocks(&ip->i_inode));
138 di->di_height = cpu_to_be16(1);
146 * gfs2_unstuff_dinode - Unstuff a dinode when the data has grown too big
147 * @ip: The GFS2 inode to unstuff
149 * This routine unstuffs a dinode and returns it to a "normal" state such
150 * that the height can be grown in the traditional way.
155 int gfs2_unstuff_dinode(struct gfs2_inode *ip)
157 struct inode *inode = &ip->i_inode;
161 down_write(&ip->i_rw_mutex);
162 folio = filemap_grab_folio(inode->i_mapping, 0);
163 error = PTR_ERR(folio);
166 error = __gfs2_unstuff_inode(ip, folio);
170 up_write(&ip->i_rw_mutex);
175 * find_metapath - Find path through the metadata tree
176 * @sdp: The superblock
177 * @block: The disk block to look up
178 * @mp: The metapath to return the result in
179 * @height: The pre-calculated height of the metadata tree
181 * This routine returns a struct metapath structure that defines a path
182 * through the metadata of inode "ip" to get to block "block".
185 * Given: "ip" is a height 3 file, "offset" is 101342453, and this is a
186 * filesystem with a blocksize of 4096.
188 * find_metapath() would return a struct metapath structure set to:
189 * mp_fheight = 3, mp_list[0] = 0, mp_list[1] = 48, and mp_list[2] = 165.
191 * That means that in order to get to the block containing the byte at
192 * offset 101342453, we would load the indirect block pointed to by pointer
193 * 0 in the dinode. We would then load the indirect block pointed to by
194 * pointer 48 in that indirect block. We would then load the data block
195 * pointed to by pointer 165 in that indirect block.
197 * ----------------------------------------
202 * ----------------------------------------
206 * ----------------------------------------
210 * |0 5 6 7 8 9 0 1 2|
211 * ----------------------------------------
215 * ----------------------------------------
220 * ----------------------------------------
224 * ----------------------------------------
225 * | Data block containing offset |
229 * ----------------------------------------
233 static void find_metapath(const struct gfs2_sbd *sdp, u64 block,
234 struct metapath *mp, unsigned int height)
238 mp->mp_fheight = height;
239 for (i = height; i--;)
240 mp->mp_list[i] = do_div(block, sdp->sd_inptrs);
243 static inline unsigned int metapath_branch_start(const struct metapath *mp)
245 if (mp->mp_list[0] == 0)
251 * metaptr1 - Return the first possible metadata pointer in a metapath buffer
252 * @height: The metadata height (0 = dinode)
255 static inline __be64 *metaptr1(unsigned int height, const struct metapath *mp)
257 struct buffer_head *bh = mp->mp_bh[height];
259 return ((__be64 *)(bh->b_data + sizeof(struct gfs2_dinode)));
260 return ((__be64 *)(bh->b_data + sizeof(struct gfs2_meta_header)));
264 * metapointer - Return pointer to start of metadata in a buffer
265 * @height: The metadata height (0 = dinode)
268 * Return a pointer to the block number of the next height of the metadata
269 * tree given a buffer containing the pointer to the current height of the
273 static inline __be64 *metapointer(unsigned int height, const struct metapath *mp)
275 __be64 *p = metaptr1(height, mp);
276 return p + mp->mp_list[height];
279 static inline const __be64 *metaend(unsigned int height, const struct metapath *mp)
281 const struct buffer_head *bh = mp->mp_bh[height];
282 return (const __be64 *)(bh->b_data + bh->b_size);
285 static void clone_metapath(struct metapath *clone, struct metapath *mp)
290 for (hgt = 0; hgt < mp->mp_aheight; hgt++)
291 get_bh(clone->mp_bh[hgt]);
294 static void gfs2_metapath_ra(struct gfs2_glock *gl, __be64 *start, __be64 *end)
298 for (t = start; t < end; t++) {
299 struct buffer_head *rabh;
304 rabh = gfs2_getbuf(gl, be64_to_cpu(*t), CREATE);
305 if (trylock_buffer(rabh)) {
306 if (!buffer_uptodate(rabh)) {
307 rabh->b_end_io = end_buffer_read_sync;
308 submit_bh(REQ_OP_READ | REQ_RAHEAD | REQ_META |
318 static inline struct buffer_head *
319 metapath_dibh(struct metapath *mp)
324 static int __fillup_metapath(struct gfs2_inode *ip, struct metapath *mp,
325 unsigned int x, unsigned int h)
328 __be64 *ptr = metapointer(x, mp);
329 u64 dblock = be64_to_cpu(*ptr);
334 ret = gfs2_meta_buffer(ip, GFS2_METATYPE_IN, dblock, &mp->mp_bh[x + 1]);
338 mp->mp_aheight = x + 1;
343 * lookup_metapath - Walk the metadata tree to a specific point
347 * Assumes that the inode's buffer has already been looked up and
348 * hooked onto mp->mp_bh[0] and that the metapath has been initialised
349 * by find_metapath().
351 * If this function encounters part of the tree which has not been
352 * allocated, it returns the current height of the tree at the point
353 * at which it found the unallocated block. Blocks which are found are
354 * added to the mp->mp_bh[] list.
359 static int lookup_metapath(struct gfs2_inode *ip, struct metapath *mp)
361 return __fillup_metapath(ip, mp, 0, ip->i_height - 1);
365 * fillup_metapath - fill up buffers for the metadata path to a specific height
368 * @h: The height to which it should be mapped
370 * Similar to lookup_metapath, but does lookups for a range of heights
372 * Returns: error or the number of buffers filled
375 static int fillup_metapath(struct gfs2_inode *ip, struct metapath *mp, int h)
381 /* find the first buffer we need to look up. */
382 for (x = h - 1; x > 0; x--) {
387 ret = __fillup_metapath(ip, mp, x, h);
390 return mp->mp_aheight - x - 1;
393 static sector_t metapath_to_block(struct gfs2_sbd *sdp, struct metapath *mp)
395 sector_t factor = 1, block = 0;
398 for (hgt = mp->mp_fheight - 1; hgt >= 0; hgt--) {
399 if (hgt < mp->mp_aheight)
400 block += mp->mp_list[hgt] * factor;
401 factor *= sdp->sd_inptrs;
406 static void release_metapath(struct metapath *mp)
410 for (i = 0; i < GFS2_MAX_META_HEIGHT; i++) {
411 if (mp->mp_bh[i] == NULL)
413 brelse(mp->mp_bh[i]);
419 * gfs2_extent_length - Returns length of an extent of blocks
420 * @bh: The metadata block
421 * @ptr: Current position in @bh
422 * @eob: Set to 1 if we hit "end of block"
424 * Returns: The length of the extent (minimum of one block)
427 static inline unsigned int gfs2_extent_length(struct buffer_head *bh, __be64 *ptr, int *eob)
429 const __be64 *end = (__be64 *)(bh->b_data + bh->b_size);
430 const __be64 *first = ptr;
431 u64 d = be64_to_cpu(*ptr);
439 } while(be64_to_cpu(*ptr) == d);
445 enum walker_status { WALK_STOP, WALK_FOLLOW, WALK_CONTINUE };
448 * gfs2_metadata_walker - walk an indirect block
449 * @mp: Metapath to indirect block
450 * @ptrs: Number of pointers to look at
452 * When returning WALK_FOLLOW, the walker must update @mp to point at the right
453 * indirect block to follow.
455 typedef enum walker_status (*gfs2_metadata_walker)(struct metapath *mp,
459 * gfs2_walk_metadata - walk a tree of indirect blocks
461 * @mp: Starting point of walk
462 * @max_len: Maximum number of blocks to walk
463 * @walker: Called during the walk
465 * Returns 1 if the walk was stopped by @walker, 0 if we went past @max_len or
466 * past the end of metadata, and a negative error code otherwise.
469 static int gfs2_walk_metadata(struct inode *inode, struct metapath *mp,
470 u64 max_len, gfs2_metadata_walker walker)
472 struct gfs2_inode *ip = GFS2_I(inode);
473 struct gfs2_sbd *sdp = GFS2_SB(inode);
479 * The walk starts in the lowest allocated indirect block, which may be
480 * before the position indicated by @mp. Adjust @max_len accordingly
481 * to avoid a short walk.
483 for (hgt = mp->mp_fheight - 1; hgt >= mp->mp_aheight; hgt--) {
484 max_len += mp->mp_list[hgt] * factor;
485 mp->mp_list[hgt] = 0;
486 factor *= sdp->sd_inptrs;
490 u16 start = mp->mp_list[hgt];
491 enum walker_status status;
495 /* Walk indirect block. */
496 ptrs = (hgt >= 1 ? sdp->sd_inptrs : sdp->sd_diptrs) - start;
499 ptrs = DIV_ROUND_UP_ULL(max_len, factor);
500 status = walker(mp, ptrs);
505 BUG_ON(mp->mp_aheight == mp->mp_fheight);
506 ptrs = mp->mp_list[hgt] - start;
515 if (status == WALK_FOLLOW)
516 goto fill_up_metapath;
519 /* Decrease height of metapath. */
520 brelse(mp->mp_bh[hgt]);
521 mp->mp_bh[hgt] = NULL;
522 mp->mp_list[hgt] = 0;
526 factor *= sdp->sd_inptrs;
528 /* Advance in metadata tree. */
529 (mp->mp_list[hgt])++;
531 if (mp->mp_list[hgt] >= sdp->sd_inptrs)
534 if (mp->mp_list[hgt] >= sdp->sd_diptrs)
539 /* Increase height of metapath. */
540 ret = fillup_metapath(ip, mp, ip->i_height - 1);
545 do_div(factor, sdp->sd_inptrs);
546 mp->mp_aheight = hgt + 1;
551 static enum walker_status gfs2_hole_walker(struct metapath *mp,
554 const __be64 *start, *ptr, *end;
557 hgt = mp->mp_aheight - 1;
558 start = metapointer(hgt, mp);
561 for (ptr = start; ptr < end; ptr++) {
563 mp->mp_list[hgt] += ptr - start;
564 if (mp->mp_aheight == mp->mp_fheight)
569 return WALK_CONTINUE;
573 * gfs2_hole_size - figure out the size of a hole
575 * @lblock: The logical starting block number
576 * @len: How far to look (in blocks)
577 * @mp: The metapath at lblock
578 * @iomap: The iomap to store the hole size in
580 * This function modifies @mp.
582 * Returns: errno on error
584 static int gfs2_hole_size(struct inode *inode, sector_t lblock, u64 len,
585 struct metapath *mp, struct iomap *iomap)
587 struct metapath clone;
591 clone_metapath(&clone, mp);
592 ret = gfs2_walk_metadata(inode, &clone, len, gfs2_hole_walker);
597 hole_size = metapath_to_block(GFS2_SB(inode), &clone) - lblock;
600 iomap->length = hole_size << inode->i_blkbits;
604 release_metapath(&clone);
608 static inline void gfs2_indirect_init(struct metapath *mp,
609 struct gfs2_glock *gl, unsigned int i,
610 unsigned offset, u64 bn)
612 __be64 *ptr = (__be64 *)(mp->mp_bh[i - 1]->b_data +
613 ((i > 1) ? sizeof(struct gfs2_meta_header) :
614 sizeof(struct gfs2_dinode)));
616 BUG_ON(mp->mp_bh[i] != NULL);
617 mp->mp_bh[i] = gfs2_meta_new(gl, bn);
618 gfs2_trans_add_meta(gl, mp->mp_bh[i]);
619 gfs2_metatype_set(mp->mp_bh[i], GFS2_METATYPE_IN, GFS2_FORMAT_IN);
620 gfs2_buffer_clear_tail(mp->mp_bh[i], sizeof(struct gfs2_meta_header));
622 *ptr = cpu_to_be64(bn);
627 ALLOC_GROW_DEPTH = 1,
628 ALLOC_GROW_HEIGHT = 2,
629 /* ALLOC_UNSTUFF = 3, TBD and rather complicated */
633 * __gfs2_iomap_alloc - Build a metadata tree of the requested height
634 * @inode: The GFS2 inode
635 * @iomap: The iomap structure
636 * @mp: The metapath, with proper height information calculated
638 * In this routine we may have to alloc:
639 * i) Indirect blocks to grow the metadata tree height
640 * ii) Indirect blocks to fill in lower part of the metadata tree
643 * This function is called after __gfs2_iomap_get, which works out the
644 * total number of blocks which we need via gfs2_alloc_size.
646 * We then do the actual allocation asking for an extent at a time (if
647 * enough contiguous free blocks are available, there will only be one
648 * allocation request per call) and uses the state machine to initialise
649 * the blocks in order.
651 * Right now, this function will allocate at most one indirect block
652 * worth of data -- with a default block size of 4K, that's slightly
653 * less than 2M. If this limitation is ever removed to allow huge
654 * allocations, we would probably still want to limit the iomap size we
655 * return to avoid stalling other tasks during huge writes; the next
656 * iomap iteration would then find the blocks already allocated.
658 * Returns: errno on error
661 static int __gfs2_iomap_alloc(struct inode *inode, struct iomap *iomap,
664 struct gfs2_inode *ip = GFS2_I(inode);
665 struct gfs2_sbd *sdp = GFS2_SB(inode);
666 struct buffer_head *dibh = metapath_dibh(mp);
668 unsigned n, i, blks, alloced = 0, iblks = 0, branch_start = 0;
669 size_t dblks = iomap->length >> inode->i_blkbits;
670 const unsigned end_of_metadata = mp->mp_fheight - 1;
672 enum alloc_state state;
676 BUG_ON(mp->mp_aheight < 1);
677 BUG_ON(dibh == NULL);
680 gfs2_trans_add_meta(ip->i_gl, dibh);
682 down_write(&ip->i_rw_mutex);
684 if (mp->mp_fheight == mp->mp_aheight) {
685 /* Bottom indirect block exists */
688 /* Need to allocate indirect blocks */
689 if (mp->mp_fheight == ip->i_height) {
690 /* Writing into existing tree, extend tree down */
691 iblks = mp->mp_fheight - mp->mp_aheight;
692 state = ALLOC_GROW_DEPTH;
694 /* Building up tree height */
695 state = ALLOC_GROW_HEIGHT;
696 iblks = mp->mp_fheight - ip->i_height;
697 branch_start = metapath_branch_start(mp);
698 iblks += (mp->mp_fheight - branch_start);
702 /* start of the second part of the function (state machine) */
704 blks = dblks + iblks;
708 ret = gfs2_alloc_blocks(ip, &bn, &n, 0);
712 if (state != ALLOC_DATA || gfs2_is_jdata(ip))
713 gfs2_trans_remove_revoke(sdp, bn, n);
715 /* Growing height of tree */
716 case ALLOC_GROW_HEIGHT:
718 ptr = (__be64 *)(dibh->b_data +
719 sizeof(struct gfs2_dinode));
722 for (; i - 1 < mp->mp_fheight - ip->i_height && n > 0;
724 gfs2_indirect_init(mp, ip->i_gl, i, 0, bn++);
725 if (i - 1 == mp->mp_fheight - ip->i_height) {
727 gfs2_buffer_copy_tail(mp->mp_bh[i],
728 sizeof(struct gfs2_meta_header),
729 dibh, sizeof(struct gfs2_dinode));
730 gfs2_buffer_clear_tail(dibh,
731 sizeof(struct gfs2_dinode) +
733 ptr = (__be64 *)(mp->mp_bh[i]->b_data +
734 sizeof(struct gfs2_meta_header));
736 state = ALLOC_GROW_DEPTH;
737 for(i = branch_start; i < mp->mp_fheight; i++) {
738 if (mp->mp_bh[i] == NULL)
740 brelse(mp->mp_bh[i]);
747 fallthrough; /* To branching from existing tree */
748 case ALLOC_GROW_DEPTH:
749 if (i > 1 && i < mp->mp_fheight)
750 gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[i-1]);
751 for (; i < mp->mp_fheight && n > 0; i++, n--)
752 gfs2_indirect_init(mp, ip->i_gl, i,
753 mp->mp_list[i-1], bn++);
754 if (i == mp->mp_fheight)
758 fallthrough; /* To tree complete, adding data blocks */
761 BUG_ON(mp->mp_bh[end_of_metadata] == NULL);
762 gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[end_of_metadata]);
764 ptr = metapointer(end_of_metadata, mp);
765 iomap->addr = bn << inode->i_blkbits;
766 iomap->flags |= IOMAP_F_MERGED | IOMAP_F_NEW;
768 *ptr++ = cpu_to_be64(bn++);
771 } while (iomap->addr == IOMAP_NULL_ADDR);
773 iomap->type = IOMAP_MAPPED;
774 iomap->length = (u64)dblks << inode->i_blkbits;
775 ip->i_height = mp->mp_fheight;
776 gfs2_add_inode_blocks(&ip->i_inode, alloced);
777 gfs2_dinode_out(ip, dibh->b_data);
779 up_write(&ip->i_rw_mutex);
783 #define IOMAP_F_GFS2_BOUNDARY IOMAP_F_PRIVATE
786 * gfs2_alloc_size - Compute the maximum allocation size
789 * @size: Requested size in blocks
791 * Compute the maximum size of the next allocation at @mp.
793 * Returns: size in blocks
795 static u64 gfs2_alloc_size(struct inode *inode, struct metapath *mp, u64 size)
797 struct gfs2_inode *ip = GFS2_I(inode);
798 struct gfs2_sbd *sdp = GFS2_SB(inode);
799 const __be64 *first, *ptr, *end;
802 * For writes to stuffed files, this function is called twice via
803 * __gfs2_iomap_get, before and after unstuffing. The size we return the
804 * first time needs to be large enough to get the reservation and
805 * allocation sizes right. The size we return the second time must
806 * be exact or else __gfs2_iomap_alloc won't do the right thing.
809 if (gfs2_is_stuffed(ip) || mp->mp_fheight != mp->mp_aheight) {
810 unsigned int maxsize = mp->mp_fheight > 1 ?
811 sdp->sd_inptrs : sdp->sd_diptrs;
812 maxsize -= mp->mp_list[mp->mp_fheight - 1];
818 first = metapointer(ip->i_height - 1, mp);
819 end = metaend(ip->i_height - 1, mp);
820 if (end - first > size)
822 for (ptr = first; ptr < end; ptr++) {
830 * __gfs2_iomap_get - Map blocks from an inode to disk blocks
832 * @pos: Starting position in bytes
833 * @length: Length to map, in bytes
834 * @flags: iomap flags
835 * @iomap: The iomap structure
840 static int __gfs2_iomap_get(struct inode *inode, loff_t pos, loff_t length,
841 unsigned flags, struct iomap *iomap,
844 struct gfs2_inode *ip = GFS2_I(inode);
845 struct gfs2_sbd *sdp = GFS2_SB(inode);
846 loff_t size = i_size_read(inode);
849 sector_t lblock_stop;
853 struct buffer_head *dibh = NULL, *bh;
859 down_read(&ip->i_rw_mutex);
861 ret = gfs2_meta_inode_buffer(ip, &dibh);
866 if (gfs2_is_stuffed(ip)) {
867 if (flags & IOMAP_WRITE) {
868 loff_t max_size = gfs2_max_stuffed_size(ip);
870 if (pos + length > max_size)
872 iomap->length = max_size;
875 if (flags & IOMAP_REPORT) {
880 iomap->length = length;
884 iomap->length = size;
886 iomap->addr = (ip->i_no_addr << inode->i_blkbits) +
887 sizeof(struct gfs2_dinode);
888 iomap->type = IOMAP_INLINE;
889 iomap->inline_data = dibh->b_data + sizeof(struct gfs2_dinode);
894 lblock = pos >> inode->i_blkbits;
895 iomap->offset = lblock << inode->i_blkbits;
896 lblock_stop = (pos + length - 1) >> inode->i_blkbits;
897 len = lblock_stop - lblock + 1;
898 iomap->length = len << inode->i_blkbits;
900 height = ip->i_height;
901 while ((lblock + 1) * sdp->sd_sb.sb_bsize > sdp->sd_heightsize[height])
903 find_metapath(sdp, lblock, mp, height);
904 if (height > ip->i_height || gfs2_is_stuffed(ip))
907 ret = lookup_metapath(ip, mp);
911 if (mp->mp_aheight != ip->i_height)
914 ptr = metapointer(ip->i_height - 1, mp);
918 bh = mp->mp_bh[ip->i_height - 1];
919 len = gfs2_extent_length(bh, ptr, &eob);
921 iomap->addr = be64_to_cpu(*ptr) << inode->i_blkbits;
922 iomap->length = len << inode->i_blkbits;
923 iomap->type = IOMAP_MAPPED;
924 iomap->flags |= IOMAP_F_MERGED;
926 iomap->flags |= IOMAP_F_GFS2_BOUNDARY;
929 iomap->bdev = inode->i_sb->s_bdev;
931 up_read(&ip->i_rw_mutex);
935 if (flags & IOMAP_REPORT) {
938 else if (height == ip->i_height)
939 ret = gfs2_hole_size(inode, lblock, len, mp, iomap);
941 iomap->length = size - iomap->offset;
942 } else if (flags & IOMAP_WRITE) {
945 if (flags & IOMAP_DIRECT)
946 goto out; /* (see gfs2_file_direct_write) */
948 len = gfs2_alloc_size(inode, mp, len);
949 alloc_size = len << inode->i_blkbits;
950 if (alloc_size < iomap->length)
951 iomap->length = alloc_size;
953 if (pos < size && height == ip->i_height)
954 ret = gfs2_hole_size(inode, lblock, len, mp, iomap);
957 iomap->addr = IOMAP_NULL_ADDR;
958 iomap->type = IOMAP_HOLE;
962 static struct folio *
963 gfs2_iomap_get_folio(struct iomap_iter *iter, loff_t pos, unsigned len)
965 struct inode *inode = iter->inode;
966 unsigned int blockmask = i_blocksize(inode) - 1;
967 struct gfs2_sbd *sdp = GFS2_SB(inode);
972 blocks = ((pos & blockmask) + len + blockmask) >> inode->i_blkbits;
973 status = gfs2_trans_begin(sdp, RES_DINODE + blocks, 0);
975 return ERR_PTR(status);
977 folio = iomap_get_folio(iter, pos, len);
983 static void gfs2_iomap_put_folio(struct inode *inode, loff_t pos,
984 unsigned copied, struct folio *folio)
986 struct gfs2_trans *tr = current->journal_info;
987 struct gfs2_inode *ip = GFS2_I(inode);
988 struct gfs2_sbd *sdp = GFS2_SB(inode);
990 if (!gfs2_is_stuffed(ip))
991 gfs2_trans_add_databufs(ip, folio, offset_in_folio(folio, pos),
997 if (tr->tr_num_buf_new)
998 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1000 gfs2_trans_end(sdp);
1003 static const struct iomap_folio_ops gfs2_iomap_folio_ops = {
1004 .get_folio = gfs2_iomap_get_folio,
1005 .put_folio = gfs2_iomap_put_folio,
1008 static int gfs2_iomap_begin_write(struct inode *inode, loff_t pos,
1009 loff_t length, unsigned flags,
1010 struct iomap *iomap,
1011 struct metapath *mp)
1013 struct gfs2_inode *ip = GFS2_I(inode);
1014 struct gfs2_sbd *sdp = GFS2_SB(inode);
1018 unstuff = gfs2_is_stuffed(ip) &&
1019 pos + length > gfs2_max_stuffed_size(ip);
1021 if (unstuff || iomap->type == IOMAP_HOLE) {
1022 unsigned int data_blocks, ind_blocks;
1023 struct gfs2_alloc_parms ap = {};
1024 unsigned int rblocks;
1025 struct gfs2_trans *tr;
1027 gfs2_write_calc_reserv(ip, iomap->length, &data_blocks,
1029 ap.target = data_blocks + ind_blocks;
1030 ret = gfs2_quota_lock_check(ip, &ap);
1034 ret = gfs2_inplace_reserve(ip, &ap);
1038 rblocks = RES_DINODE + ind_blocks;
1039 if (gfs2_is_jdata(ip))
1040 rblocks += data_blocks;
1041 if (ind_blocks || data_blocks)
1042 rblocks += RES_STATFS + RES_QUOTA;
1043 if (inode == sdp->sd_rindex)
1044 rblocks += 2 * RES_STATFS;
1045 rblocks += gfs2_rg_blocks(ip, data_blocks + ind_blocks);
1047 ret = gfs2_trans_begin(sdp, rblocks,
1048 iomap->length >> inode->i_blkbits);
1050 goto out_trans_fail;
1053 ret = gfs2_unstuff_dinode(ip);
1056 release_metapath(mp);
1057 ret = __gfs2_iomap_get(inode, iomap->offset,
1058 iomap->length, flags, iomap, mp);
1063 if (iomap->type == IOMAP_HOLE) {
1064 ret = __gfs2_iomap_alloc(inode, iomap, mp);
1066 gfs2_trans_end(sdp);
1067 gfs2_inplace_release(ip);
1068 punch_hole(ip, iomap->offset, iomap->length);
1073 tr = current->journal_info;
1074 if (tr->tr_num_buf_new)
1075 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1077 gfs2_trans_end(sdp);
1080 if (gfs2_is_stuffed(ip) || gfs2_is_jdata(ip))
1081 iomap->folio_ops = &gfs2_iomap_folio_ops;
1085 gfs2_trans_end(sdp);
1087 gfs2_inplace_release(ip);
1089 gfs2_quota_unlock(ip);
1093 static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
1094 unsigned flags, struct iomap *iomap,
1095 struct iomap *srcmap)
1097 struct gfs2_inode *ip = GFS2_I(inode);
1098 struct metapath mp = { .mp_aheight = 1, };
1101 if (gfs2_is_jdata(ip))
1102 iomap->flags |= IOMAP_F_BUFFER_HEAD;
1104 trace_gfs2_iomap_start(ip, pos, length, flags);
1105 ret = __gfs2_iomap_get(inode, pos, length, flags, iomap, &mp);
1109 switch(flags & (IOMAP_WRITE | IOMAP_ZERO)) {
1111 if (flags & IOMAP_DIRECT) {
1113 * Silently fall back to buffered I/O for stuffed files
1114 * or if we've got a hole (see gfs2_file_direct_write).
1116 if (iomap->type != IOMAP_MAPPED)
1122 if (iomap->type == IOMAP_HOLE)
1129 ret = gfs2_iomap_begin_write(inode, pos, length, flags, iomap, &mp);
1132 release_metapath(&mp);
1133 trace_gfs2_iomap_end(ip, iomap, ret);
1137 static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length,
1138 ssize_t written, unsigned flags, struct iomap *iomap)
1140 struct gfs2_inode *ip = GFS2_I(inode);
1141 struct gfs2_sbd *sdp = GFS2_SB(inode);
1143 switch (flags & (IOMAP_WRITE | IOMAP_ZERO)) {
1145 if (flags & IOMAP_DIRECT)
1149 if (iomap->type == IOMAP_HOLE)
1156 if (!gfs2_is_stuffed(ip))
1157 gfs2_ordered_add_inode(ip);
1159 if (inode == sdp->sd_rindex)
1160 adjust_fs_space(inode);
1162 gfs2_inplace_release(ip);
1164 if (ip->i_qadata && ip->i_qadata->qa_qd_num)
1165 gfs2_quota_unlock(ip);
1167 if (length != written && (iomap->flags & IOMAP_F_NEW)) {
1168 /* Deallocate blocks that were just allocated. */
1169 loff_t hstart = round_up(pos + written, i_blocksize(inode));
1170 loff_t hend = iomap->offset + iomap->length;
1172 if (hstart < hend) {
1173 truncate_pagecache_range(inode, hstart, hend - 1);
1174 punch_hole(ip, hstart, hend - hstart);
1178 if (unlikely(!written))
1181 if (iomap->flags & IOMAP_F_SIZE_CHANGED)
1182 mark_inode_dirty(inode);
1183 set_bit(GLF_DIRTY, &ip->i_gl->gl_flags);
1187 const struct iomap_ops gfs2_iomap_ops = {
1188 .iomap_begin = gfs2_iomap_begin,
1189 .iomap_end = gfs2_iomap_end,
1193 * gfs2_block_map - Map one or more blocks of an inode to a disk block
1195 * @lblock: The logical block number
1196 * @bh_map: The bh to be mapped
1197 * @create: True if its ok to alloc blocks to satify the request
1199 * The size of the requested mapping is defined in bh_map->b_size.
1201 * Clears buffer_mapped(bh_map) and leaves bh_map->b_size unchanged
1202 * when @lblock is not mapped. Sets buffer_mapped(bh_map) and
1203 * bh_map->b_size to indicate the size of the mapping when @lblock and
1204 * successive blocks are mapped, up to the requested size.
1206 * Sets buffer_boundary() if a read of metadata will be required
1207 * before the next block can be mapped. Sets buffer_new() if new
1208 * blocks were allocated.
1213 int gfs2_block_map(struct inode *inode, sector_t lblock,
1214 struct buffer_head *bh_map, int create)
1216 struct gfs2_inode *ip = GFS2_I(inode);
1217 loff_t pos = (loff_t)lblock << inode->i_blkbits;
1218 loff_t length = bh_map->b_size;
1219 struct iomap iomap = { };
1222 clear_buffer_mapped(bh_map);
1223 clear_buffer_new(bh_map);
1224 clear_buffer_boundary(bh_map);
1225 trace_gfs2_bmap(ip, bh_map, lblock, create, 1);
1228 ret = gfs2_iomap_get(inode, pos, length, &iomap);
1230 ret = gfs2_iomap_alloc(inode, pos, length, &iomap);
1234 if (iomap.length > bh_map->b_size) {
1235 iomap.length = bh_map->b_size;
1236 iomap.flags &= ~IOMAP_F_GFS2_BOUNDARY;
1238 if (iomap.addr != IOMAP_NULL_ADDR)
1239 map_bh(bh_map, inode->i_sb, iomap.addr >> inode->i_blkbits);
1240 bh_map->b_size = iomap.length;
1241 if (iomap.flags & IOMAP_F_GFS2_BOUNDARY)
1242 set_buffer_boundary(bh_map);
1243 if (iomap.flags & IOMAP_F_NEW)
1244 set_buffer_new(bh_map);
1247 trace_gfs2_bmap(ip, bh_map, lblock, create, ret);
1251 int gfs2_get_extent(struct inode *inode, u64 lblock, u64 *dblock,
1252 unsigned int *extlen)
1254 unsigned int blkbits = inode->i_blkbits;
1255 struct iomap iomap = { };
1259 ret = gfs2_iomap_get(inode, lblock << blkbits, *extlen << blkbits,
1263 if (iomap.type != IOMAP_MAPPED)
1265 *dblock = iomap.addr >> blkbits;
1266 len = iomap.length >> blkbits;
1272 int gfs2_alloc_extent(struct inode *inode, u64 lblock, u64 *dblock,
1273 unsigned int *extlen, bool *new)
1275 unsigned int blkbits = inode->i_blkbits;
1276 struct iomap iomap = { };
1280 ret = gfs2_iomap_alloc(inode, lblock << blkbits, *extlen << blkbits,
1284 if (iomap.type != IOMAP_MAPPED)
1286 *dblock = iomap.addr >> blkbits;
1287 len = iomap.length >> blkbits;
1290 *new = iomap.flags & IOMAP_F_NEW;
1295 * NOTE: Never call gfs2_block_zero_range with an open transaction because it
1296 * uses iomap write to perform its actions, which begin their own transactions
1297 * (iomap_begin, get_folio, etc.)
1299 static int gfs2_block_zero_range(struct inode *inode, loff_t from,
1300 unsigned int length)
1302 BUG_ON(current->journal_info);
1303 return iomap_zero_range(inode, from, length, NULL, &gfs2_iomap_ops);
1306 #define GFS2_JTRUNC_REVOKES 8192
1309 * gfs2_journaled_truncate - Wrapper for truncate_pagecache for jdata files
1310 * @inode: The inode being truncated
1311 * @oldsize: The original (larger) size
1312 * @newsize: The new smaller size
1314 * With jdata files, we have to journal a revoke for each block which is
1315 * truncated. As a result, we need to split this into separate transactions
1316 * if the number of pages being truncated gets too large.
1319 static int gfs2_journaled_truncate(struct inode *inode, u64 oldsize, u64 newsize)
1321 struct gfs2_sbd *sdp = GFS2_SB(inode);
1322 u64 max_chunk = GFS2_JTRUNC_REVOKES * sdp->sd_vfs->s_blocksize;
1326 while (oldsize != newsize) {
1327 struct gfs2_trans *tr;
1330 chunk = oldsize - newsize;
1331 if (chunk > max_chunk)
1334 offs = oldsize & ~PAGE_MASK;
1335 if (offs && chunk > PAGE_SIZE)
1336 chunk = offs + ((chunk - offs) & PAGE_MASK);
1338 truncate_pagecache(inode, oldsize - chunk);
1341 tr = current->journal_info;
1342 if (!test_bit(TR_TOUCHED, &tr->tr_flags))
1345 gfs2_trans_end(sdp);
1346 error = gfs2_trans_begin(sdp, RES_DINODE, GFS2_JTRUNC_REVOKES);
1354 static int trunc_start(struct inode *inode, u64 newsize)
1356 struct gfs2_inode *ip = GFS2_I(inode);
1357 struct gfs2_sbd *sdp = GFS2_SB(inode);
1358 struct buffer_head *dibh = NULL;
1359 int journaled = gfs2_is_jdata(ip);
1360 u64 oldsize = inode->i_size;
1363 if (!gfs2_is_stuffed(ip)) {
1364 unsigned int blocksize = i_blocksize(inode);
1365 unsigned int offs = newsize & (blocksize - 1);
1367 error = gfs2_block_zero_range(inode, newsize,
1374 error = gfs2_trans_begin(sdp, RES_DINODE + RES_JDATA, GFS2_JTRUNC_REVOKES);
1376 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
1380 error = gfs2_meta_inode_buffer(ip, &dibh);
1384 gfs2_trans_add_meta(ip->i_gl, dibh);
1386 if (gfs2_is_stuffed(ip))
1387 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode) + newsize);
1389 ip->i_diskflags |= GFS2_DIF_TRUNC_IN_PROG;
1391 i_size_write(inode, newsize);
1392 inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode));
1393 gfs2_dinode_out(ip, dibh->b_data);
1396 error = gfs2_journaled_truncate(inode, oldsize, newsize);
1398 truncate_pagecache(inode, newsize);
1402 if (current->journal_info)
1403 gfs2_trans_end(sdp);
1407 int gfs2_iomap_get(struct inode *inode, loff_t pos, loff_t length,
1408 struct iomap *iomap)
1410 struct metapath mp = { .mp_aheight = 1, };
1413 ret = __gfs2_iomap_get(inode, pos, length, 0, iomap, &mp);
1414 release_metapath(&mp);
1418 int gfs2_iomap_alloc(struct inode *inode, loff_t pos, loff_t length,
1419 struct iomap *iomap)
1421 struct metapath mp = { .mp_aheight = 1, };
1424 ret = __gfs2_iomap_get(inode, pos, length, IOMAP_WRITE, iomap, &mp);
1425 if (!ret && iomap->type == IOMAP_HOLE)
1426 ret = __gfs2_iomap_alloc(inode, iomap, &mp);
1427 release_metapath(&mp);
1432 * sweep_bh_for_rgrps - find an rgrp in a meta buffer and free blocks therein
1434 * @rd_gh: holder of resource group glock
1435 * @bh: buffer head to sweep
1436 * @start: starting point in bh
1437 * @end: end point in bh
1438 * @meta: true if bh points to metadata (rather than data)
1439 * @btotal: place to keep count of total blocks freed
1441 * We sweep a metadata buffer (provided by the metapath) for blocks we need to
1442 * free, and free them all. However, we do it one rgrp at a time. If this
1443 * block has references to multiple rgrps, we break it into individual
1444 * transactions. This allows other processes to use the rgrps while we're
1445 * focused on a single one, for better concurrency / performance.
1446 * At every transaction boundary, we rewrite the inode into the journal.
1447 * That way the bitmaps are kept consistent with the inode and we can recover
1448 * if we're interrupted by power-outages.
1450 * Returns: 0, or return code if an error occurred.
1451 * *btotal has the total number of blocks freed
1453 static int sweep_bh_for_rgrps(struct gfs2_inode *ip, struct gfs2_holder *rd_gh,
1454 struct buffer_head *bh, __be64 *start, __be64 *end,
1455 bool meta, u32 *btotal)
1457 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1458 struct gfs2_rgrpd *rgd;
1459 struct gfs2_trans *tr;
1461 int blks_outside_rgrp;
1462 u64 bn, bstart, isize_blks;
1463 s64 blen; /* needs to be s64 or gfs2_add_inode_blocks breaks */
1465 bool buf_in_tr = false; /* buffer was added to transaction */
1469 if (gfs2_holder_initialized(rd_gh)) {
1470 rgd = gfs2_glock2rgrp(rd_gh->gh_gl);
1471 gfs2_assert_withdraw(sdp,
1472 gfs2_glock_is_locked_by_me(rd_gh->gh_gl));
1474 blks_outside_rgrp = 0;
1478 for (p = start; p < end; p++) {
1481 bn = be64_to_cpu(*p);
1484 if (!rgrp_contains_block(rgd, bn)) {
1485 blks_outside_rgrp++;
1489 rgd = gfs2_blk2rgrpd(sdp, bn, true);
1490 if (unlikely(!rgd)) {
1494 ret = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE,
1495 LM_FLAG_NODE_SCOPE, rd_gh);
1499 /* Must be done with the rgrp glock held: */
1500 if (gfs2_rs_active(&ip->i_res) &&
1501 rgd == ip->i_res.rs_rgd)
1502 gfs2_rs_deltree(&ip->i_res);
1505 /* The size of our transactions will be unknown until we
1506 actually process all the metadata blocks that relate to
1507 the rgrp. So we estimate. We know it can't be more than
1508 the dinode's i_blocks and we don't want to exceed the
1509 journal flush threshold, sd_log_thresh2. */
1510 if (current->journal_info == NULL) {
1511 unsigned int jblocks_rqsted, revokes;
1513 jblocks_rqsted = rgd->rd_length + RES_DINODE +
1515 isize_blks = gfs2_get_inode_blocks(&ip->i_inode);
1516 if (isize_blks > atomic_read(&sdp->sd_log_thresh2))
1518 atomic_read(&sdp->sd_log_thresh2);
1520 jblocks_rqsted += isize_blks;
1521 revokes = jblocks_rqsted;
1523 revokes += end - start;
1524 else if (ip->i_depth)
1525 revokes += sdp->sd_inptrs;
1526 ret = gfs2_trans_begin(sdp, jblocks_rqsted, revokes);
1529 down_write(&ip->i_rw_mutex);
1531 /* check if we will exceed the transaction blocks requested */
1532 tr = current->journal_info;
1533 if (tr->tr_num_buf_new + RES_STATFS +
1534 RES_QUOTA >= atomic_read(&sdp->sd_log_thresh2)) {
1535 /* We set blks_outside_rgrp to ensure the loop will
1536 be repeated for the same rgrp, but with a new
1538 blks_outside_rgrp++;
1539 /* This next part is tricky. If the buffer was added
1540 to the transaction, we've already set some block
1541 pointers to 0, so we better follow through and free
1542 them, or we will introduce corruption (so break).
1543 This may be impossible, or at least rare, but I
1544 decided to cover the case regardless.
1546 If the buffer was not added to the transaction
1547 (this call), doing so would exceed our transaction
1548 size, so we need to end the transaction and start a
1549 new one (so goto). */
1556 gfs2_trans_add_meta(ip->i_gl, bh);
1559 if (bstart + blen == bn) {
1564 __gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta);
1566 gfs2_add_inode_blocks(&ip->i_inode, -blen);
1572 __gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta);
1574 gfs2_add_inode_blocks(&ip->i_inode, -blen);
1577 if (!ret && blks_outside_rgrp) { /* If buffer still has non-zero blocks
1578 outside the rgrp we just processed,
1579 do it all over again. */
1580 if (current->journal_info) {
1581 struct buffer_head *dibh;
1583 ret = gfs2_meta_inode_buffer(ip, &dibh);
1587 /* Every transaction boundary, we rewrite the dinode
1588 to keep its di_blocks current in case of failure. */
1589 inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode));
1590 gfs2_trans_add_meta(ip->i_gl, dibh);
1591 gfs2_dinode_out(ip, dibh->b_data);
1593 up_write(&ip->i_rw_mutex);
1594 gfs2_trans_end(sdp);
1597 gfs2_glock_dq_uninit(rd_gh);
1605 static bool mp_eq_to_hgt(struct metapath *mp, __u16 *list, unsigned int h)
1607 if (memcmp(mp->mp_list, list, h * sizeof(mp->mp_list[0])))
1613 * find_nonnull_ptr - find a non-null pointer given a metapath and height
1614 * @sdp: The superblock
1615 * @mp: starting metapath
1616 * @h: desired height to search
1617 * @end_list: See punch_hole().
1618 * @end_aligned: See punch_hole().
1620 * Assumes the metapath is valid (with buffers) out to height h.
1621 * Returns: true if a non-null pointer was found in the metapath buffer
1622 * false if all remaining pointers are NULL in the buffer
1624 static bool find_nonnull_ptr(struct gfs2_sbd *sdp, struct metapath *mp,
1626 __u16 *end_list, unsigned int end_aligned)
1628 struct buffer_head *bh = mp->mp_bh[h];
1629 __be64 *first, *ptr, *end;
1631 first = metaptr1(h, mp);
1632 ptr = first + mp->mp_list[h];
1633 end = (__be64 *)(bh->b_data + bh->b_size);
1634 if (end_list && mp_eq_to_hgt(mp, end_list, h)) {
1635 bool keep_end = h < end_aligned;
1636 end = first + end_list[h] + keep_end;
1640 if (*ptr) { /* if we have a non-null pointer */
1641 mp->mp_list[h] = ptr - first;
1643 if (h < GFS2_MAX_META_HEIGHT)
1652 enum dealloc_states {
1653 DEALLOC_MP_FULL = 0, /* Strip a metapath with all buffers read in */
1654 DEALLOC_MP_LOWER = 1, /* lower the metapath strip height */
1655 DEALLOC_FILL_MP = 2, /* Fill in the metapath to the given height. */
1656 DEALLOC_DONE = 3, /* process complete */
1660 metapointer_range(struct metapath *mp, int height,
1661 __u16 *start_list, unsigned int start_aligned,
1662 __u16 *end_list, unsigned int end_aligned,
1663 __be64 **start, __be64 **end)
1665 struct buffer_head *bh = mp->mp_bh[height];
1668 first = metaptr1(height, mp);
1670 if (mp_eq_to_hgt(mp, start_list, height)) {
1671 bool keep_start = height < start_aligned;
1672 *start = first + start_list[height] + keep_start;
1674 *end = (__be64 *)(bh->b_data + bh->b_size);
1675 if (end_list && mp_eq_to_hgt(mp, end_list, height)) {
1676 bool keep_end = height < end_aligned;
1677 *end = first + end_list[height] + keep_end;
1681 static inline bool walk_done(struct gfs2_sbd *sdp,
1682 struct metapath *mp, int height,
1683 __u16 *end_list, unsigned int end_aligned)
1688 bool keep_end = height < end_aligned;
1689 if (!mp_eq_to_hgt(mp, end_list, height))
1691 end = end_list[height] + keep_end;
1693 end = (height > 0) ? sdp->sd_inptrs : sdp->sd_diptrs;
1694 return mp->mp_list[height] >= end;
1698 * punch_hole - deallocate blocks in a file
1699 * @ip: inode to truncate
1700 * @offset: the start of the hole
1701 * @length: the size of the hole (or 0 for truncate)
1703 * Punch a hole into a file or truncate a file at a given position. This
1704 * function operates in whole blocks (@offset and @length are rounded
1705 * accordingly); partially filled blocks must be cleared otherwise.
1707 * This function works from the bottom up, and from the right to the left. In
1708 * other words, it strips off the highest layer (data) before stripping any of
1709 * the metadata. Doing it this way is best in case the operation is interrupted
1710 * by power failure, etc. The dinode is rewritten in every transaction to
1711 * guarantee integrity.
1713 static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length)
1715 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1716 u64 maxsize = sdp->sd_heightsize[ip->i_height];
1717 struct metapath mp = {};
1718 struct buffer_head *dibh, *bh;
1719 struct gfs2_holder rd_gh;
1720 unsigned int bsize_shift = sdp->sd_sb.sb_bsize_shift;
1721 u64 lblock = (offset + (1 << bsize_shift) - 1) >> bsize_shift;
1722 __u16 start_list[GFS2_MAX_META_HEIGHT];
1723 __u16 __end_list[GFS2_MAX_META_HEIGHT], *end_list = NULL;
1724 unsigned int start_aligned, end_aligned;
1725 unsigned int strip_h = ip->i_height - 1;
1728 int mp_h; /* metapath buffers are read in to this height */
1730 __be64 *start, *end;
1732 if (offset >= maxsize) {
1734 * The starting point lies beyond the allocated metadata;
1735 * there are no blocks to deallocate.
1741 * The start position of the hole is defined by lblock, start_list, and
1742 * start_aligned. The end position of the hole is defined by lend,
1743 * end_list, and end_aligned.
1745 * start_aligned and end_aligned define down to which height the start
1746 * and end positions are aligned to the metadata tree (i.e., the
1747 * position is a multiple of the metadata granularity at the height
1748 * above). This determines at which heights additional meta pointers
1749 * needs to be preserved for the remaining data.
1753 u64 end_offset = offset + length;
1757 * Clip the end at the maximum file size for the given height:
1758 * that's how far the metadata goes; files bigger than that
1759 * will have additional layers of indirection.
1761 if (end_offset > maxsize)
1762 end_offset = maxsize;
1763 lend = end_offset >> bsize_shift;
1768 find_metapath(sdp, lend, &mp, ip->i_height);
1769 end_list = __end_list;
1770 memcpy(end_list, mp.mp_list, sizeof(mp.mp_list));
1772 for (mp_h = ip->i_height - 1; mp_h > 0; mp_h--) {
1779 find_metapath(sdp, lblock, &mp, ip->i_height);
1780 memcpy(start_list, mp.mp_list, sizeof(start_list));
1782 for (mp_h = ip->i_height - 1; mp_h > 0; mp_h--) {
1783 if (start_list[mp_h])
1786 start_aligned = mp_h;
1788 ret = gfs2_meta_inode_buffer(ip, &dibh);
1793 ret = lookup_metapath(ip, &mp);
1797 /* issue read-ahead on metadata */
1798 for (mp_h = 0; mp_h < mp.mp_aheight - 1; mp_h++) {
1799 metapointer_range(&mp, mp_h, start_list, start_aligned,
1800 end_list, end_aligned, &start, &end);
1801 gfs2_metapath_ra(ip->i_gl, start, end);
1804 if (mp.mp_aheight == ip->i_height)
1805 state = DEALLOC_MP_FULL; /* We have a complete metapath */
1807 state = DEALLOC_FILL_MP; /* deal with partial metapath */
1809 ret = gfs2_rindex_update(sdp);
1813 ret = gfs2_quota_hold(ip, NO_UID_QUOTA_CHANGE, NO_GID_QUOTA_CHANGE);
1816 gfs2_holder_mark_uninitialized(&rd_gh);
1820 while (state != DEALLOC_DONE) {
1822 /* Truncate a full metapath at the given strip height.
1823 * Note that strip_h == mp_h in order to be in this state. */
1824 case DEALLOC_MP_FULL:
1825 bh = mp.mp_bh[mp_h];
1826 gfs2_assert_withdraw(sdp, bh);
1827 if (gfs2_assert_withdraw(sdp,
1828 prev_bnr != bh->b_blocknr)) {
1829 fs_emerg(sdp, "inode %llu, block:%llu, i_h:%u,"
1830 "s_h:%u, mp_h:%u\n",
1831 (unsigned long long)ip->i_no_addr,
1832 prev_bnr, ip->i_height, strip_h, mp_h);
1834 prev_bnr = bh->b_blocknr;
1836 if (gfs2_metatype_check(sdp, bh,
1837 (mp_h ? GFS2_METATYPE_IN :
1838 GFS2_METATYPE_DI))) {
1844 * Below, passing end_aligned as 0 gives us the
1845 * metapointer range excluding the end point: the end
1846 * point is the first metapath we must not deallocate!
1849 metapointer_range(&mp, mp_h, start_list, start_aligned,
1850 end_list, 0 /* end_aligned */,
1852 ret = sweep_bh_for_rgrps(ip, &rd_gh, mp.mp_bh[mp_h],
1854 mp_h != ip->i_height - 1,
1857 /* If we hit an error or just swept dinode buffer,
1860 state = DEALLOC_DONE;
1863 state = DEALLOC_MP_LOWER;
1866 /* lower the metapath strip height */
1867 case DEALLOC_MP_LOWER:
1868 /* We're done with the current buffer, so release it,
1869 unless it's the dinode buffer. Then back up to the
1870 previous pointer. */
1872 brelse(mp.mp_bh[mp_h]);
1873 mp.mp_bh[mp_h] = NULL;
1875 /* If we can't get any lower in height, we've stripped
1876 off all we can. Next step is to back up and start
1877 stripping the previous level of metadata. */
1880 memcpy(mp.mp_list, start_list, sizeof(start_list));
1882 state = DEALLOC_FILL_MP;
1885 mp.mp_list[mp_h] = 0;
1886 mp_h--; /* search one metadata height down */
1888 if (walk_done(sdp, &mp, mp_h, end_list, end_aligned))
1890 /* Here we've found a part of the metapath that is not
1891 * allocated. We need to search at that height for the
1892 * next non-null pointer. */
1893 if (find_nonnull_ptr(sdp, &mp, mp_h, end_list, end_aligned)) {
1894 state = DEALLOC_FILL_MP;
1897 /* No more non-null pointers at this height. Back up
1898 to the previous height and try again. */
1899 break; /* loop around in the same state */
1901 /* Fill the metapath with buffers to the given height. */
1902 case DEALLOC_FILL_MP:
1903 /* Fill the buffers out to the current height. */
1904 ret = fillup_metapath(ip, &mp, mp_h);
1908 /* On the first pass, issue read-ahead on metadata. */
1909 if (mp.mp_aheight > 1 && strip_h == ip->i_height - 1) {
1910 unsigned int height = mp.mp_aheight - 1;
1912 /* No read-ahead for data blocks. */
1913 if (mp.mp_aheight - 1 == strip_h)
1916 for (; height >= mp.mp_aheight - ret; height--) {
1917 metapointer_range(&mp, height,
1918 start_list, start_aligned,
1919 end_list, end_aligned,
1921 gfs2_metapath_ra(ip->i_gl, start, end);
1925 /* If buffers found for the entire strip height */
1926 if (mp.mp_aheight - 1 == strip_h) {
1927 state = DEALLOC_MP_FULL;
1930 if (mp.mp_aheight < ip->i_height) /* We have a partial height */
1931 mp_h = mp.mp_aheight - 1;
1933 /* If we find a non-null block pointer, crawl a bit
1934 higher up in the metapath and try again, otherwise
1935 we need to look lower for a new starting point. */
1936 if (find_nonnull_ptr(sdp, &mp, mp_h, end_list, end_aligned))
1939 state = DEALLOC_MP_LOWER;
1945 if (current->journal_info == NULL) {
1946 ret = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS +
1950 down_write(&ip->i_rw_mutex);
1952 gfs2_statfs_change(sdp, 0, +btotal, 0);
1953 gfs2_quota_change(ip, -(s64)btotal, ip->i_inode.i_uid,
1955 inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode));
1956 gfs2_trans_add_meta(ip->i_gl, dibh);
1957 gfs2_dinode_out(ip, dibh->b_data);
1958 up_write(&ip->i_rw_mutex);
1959 gfs2_trans_end(sdp);
1963 if (gfs2_holder_initialized(&rd_gh))
1964 gfs2_glock_dq_uninit(&rd_gh);
1965 if (current->journal_info) {
1966 up_write(&ip->i_rw_mutex);
1967 gfs2_trans_end(sdp);
1970 gfs2_quota_unhold(ip);
1972 release_metapath(&mp);
1976 static int trunc_end(struct gfs2_inode *ip)
1978 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1979 struct buffer_head *dibh;
1982 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
1986 down_write(&ip->i_rw_mutex);
1988 error = gfs2_meta_inode_buffer(ip, &dibh);
1992 if (!i_size_read(&ip->i_inode)) {
1994 ip->i_goal = ip->i_no_addr;
1995 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
1996 gfs2_ordered_del_inode(ip);
1998 inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode));
1999 ip->i_diskflags &= ~GFS2_DIF_TRUNC_IN_PROG;
2001 gfs2_trans_add_meta(ip->i_gl, dibh);
2002 gfs2_dinode_out(ip, dibh->b_data);
2006 up_write(&ip->i_rw_mutex);
2007 gfs2_trans_end(sdp);
2012 * do_shrink - make a file smaller
2014 * @newsize: the size to make the file
2016 * Called with an exclusive lock on @inode. The @size must
2017 * be equal to or smaller than the current inode size.
2022 static int do_shrink(struct inode *inode, u64 newsize)
2024 struct gfs2_inode *ip = GFS2_I(inode);
2027 error = trunc_start(inode, newsize);
2030 if (gfs2_is_stuffed(ip))
2033 error = punch_hole(ip, newsize, 0);
2035 error = trunc_end(ip);
2041 * do_grow - Touch and update inode size
2043 * @size: The new size
2045 * This function updates the timestamps on the inode and
2046 * may also increase the size of the inode. This function
2047 * must not be called with @size any smaller than the current
2050 * Although it is not strictly required to unstuff files here,
2051 * earlier versions of GFS2 have a bug in the stuffed file reading
2052 * code which will result in a buffer overrun if the size is larger
2053 * than the max stuffed file size. In order to prevent this from
2054 * occurring, such files are unstuffed, but in other cases we can
2055 * just update the inode size directly.
2057 * Returns: 0 on success, or -ve on error
2060 static int do_grow(struct inode *inode, u64 size)
2062 struct gfs2_inode *ip = GFS2_I(inode);
2063 struct gfs2_sbd *sdp = GFS2_SB(inode);
2064 struct gfs2_alloc_parms ap = { .target = 1, };
2065 struct buffer_head *dibh;
2069 if (gfs2_is_stuffed(ip) && size > gfs2_max_stuffed_size(ip)) {
2070 error = gfs2_quota_lock_check(ip, &ap);
2074 error = gfs2_inplace_reserve(ip, &ap);
2076 goto do_grow_qunlock;
2080 error = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS + RES_RG_BIT +
2082 gfs2_is_jdata(ip) ? RES_JDATA : 0) +
2083 (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF ?
2086 goto do_grow_release;
2089 error = gfs2_unstuff_dinode(ip);
2094 error = gfs2_meta_inode_buffer(ip, &dibh);
2098 truncate_setsize(inode, size);
2099 inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode));
2100 gfs2_trans_add_meta(ip->i_gl, dibh);
2101 gfs2_dinode_out(ip, dibh->b_data);
2105 gfs2_trans_end(sdp);
2108 gfs2_inplace_release(ip);
2110 gfs2_quota_unlock(ip);
2116 * gfs2_setattr_size - make a file a given size
2118 * @newsize: the size to make the file
2120 * The file size can grow, shrink, or stay the same size. This
2121 * is called holding i_rwsem and an exclusive glock on the inode
2127 int gfs2_setattr_size(struct inode *inode, u64 newsize)
2129 struct gfs2_inode *ip = GFS2_I(inode);
2132 BUG_ON(!S_ISREG(inode->i_mode));
2134 ret = inode_newsize_ok(inode, newsize);
2138 inode_dio_wait(inode);
2140 ret = gfs2_qa_get(ip);
2144 if (newsize >= inode->i_size) {
2145 ret = do_grow(inode, newsize);
2149 ret = do_shrink(inode, newsize);
2156 int gfs2_truncatei_resume(struct gfs2_inode *ip)
2159 error = punch_hole(ip, i_size_read(&ip->i_inode), 0);
2161 error = trunc_end(ip);
2165 int gfs2_file_dealloc(struct gfs2_inode *ip)
2167 return punch_hole(ip, 0, 0);
2171 * gfs2_free_journal_extents - Free cached journal bmap info
2176 void gfs2_free_journal_extents(struct gfs2_jdesc *jd)
2178 struct gfs2_journal_extent *jext;
2180 while(!list_empty(&jd->extent_list)) {
2181 jext = list_first_entry(&jd->extent_list, struct gfs2_journal_extent, list);
2182 list_del(&jext->list);
2188 * gfs2_add_jextent - Add or merge a new extent to extent cache
2189 * @jd: The journal descriptor
2190 * @lblock: The logical block at start of new extent
2191 * @dblock: The physical block at start of new extent
2192 * @blocks: Size of extent in fs blocks
2194 * Returns: 0 on success or -ENOMEM
2197 static int gfs2_add_jextent(struct gfs2_jdesc *jd, u64 lblock, u64 dblock, u64 blocks)
2199 struct gfs2_journal_extent *jext;
2201 if (!list_empty(&jd->extent_list)) {
2202 jext = list_last_entry(&jd->extent_list, struct gfs2_journal_extent, list);
2203 if ((jext->dblock + jext->blocks) == dblock) {
2204 jext->blocks += blocks;
2209 jext = kzalloc(sizeof(struct gfs2_journal_extent), GFP_NOFS);
2212 jext->dblock = dblock;
2213 jext->lblock = lblock;
2214 jext->blocks = blocks;
2215 list_add_tail(&jext->list, &jd->extent_list);
2221 * gfs2_map_journal_extents - Cache journal bmap info
2222 * @sdp: The super block
2223 * @jd: The journal to map
2225 * Create a reusable "extent" mapping from all logical
2226 * blocks to all physical blocks for the given journal. This will save
2227 * us time when writing journal blocks. Most journals will have only one
2228 * extent that maps all their logical blocks. That's because gfs2.mkfs
2229 * arranges the journal blocks sequentially to maximize performance.
2230 * So the extent would map the first block for the entire file length.
2231 * However, gfs2_jadd can happen while file activity is happening, so
2232 * those journals may not be sequential. Less likely is the case where
2233 * the users created their own journals by mounting the metafs and
2234 * laying it out. But it's still possible. These journals might have
2237 * Returns: 0 on success, or error on failure
2240 int gfs2_map_journal_extents(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd)
2244 struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
2245 struct buffer_head bh;
2246 unsigned int shift = sdp->sd_sb.sb_bsize_shift;
2251 start = ktime_get();
2252 lblock_stop = i_size_read(jd->jd_inode) >> shift;
2253 size = (lblock_stop - lblock) << shift;
2255 WARN_ON(!list_empty(&jd->extent_list));
2261 rc = gfs2_block_map(jd->jd_inode, lblock, &bh, 0);
2262 if (rc || !buffer_mapped(&bh))
2264 rc = gfs2_add_jextent(jd, lblock, bh.b_blocknr, bh.b_size >> shift);
2268 lblock += (bh.b_size >> ip->i_inode.i_blkbits);
2272 fs_info(sdp, "journal %d mapped with %u extents in %lldms\n", jd->jd_jid,
2273 jd->nr_extents, ktime_ms_delta(end, start));
2277 fs_warn(sdp, "error %d mapping journal %u at offset %llu (extent %u)\n",
2279 (unsigned long long)(i_size_read(jd->jd_inode) - size),
2281 fs_warn(sdp, "bmap=%d lblock=%llu block=%llu, state=0x%08lx, size=%llu\n",
2282 rc, (unsigned long long)lblock, (unsigned long long)bh.b_blocknr,
2283 bh.b_state, (unsigned long long)bh.b_size);
2284 gfs2_free_journal_extents(jd);
2289 * gfs2_write_alloc_required - figure out if a write will require an allocation
2290 * @ip: the file being written to
2291 * @offset: the offset to write to
2292 * @len: the number of bytes being written
2294 * Returns: 1 if an alloc is required, 0 otherwise
2297 int gfs2_write_alloc_required(struct gfs2_inode *ip, u64 offset,
2300 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2301 struct buffer_head bh;
2303 u64 lblock, lblock_stop, size;
2309 if (gfs2_is_stuffed(ip)) {
2310 if (offset + len > gfs2_max_stuffed_size(ip))
2315 shift = sdp->sd_sb.sb_bsize_shift;
2316 BUG_ON(gfs2_is_dir(ip));
2317 end_of_file = (i_size_read(&ip->i_inode) + sdp->sd_sb.sb_bsize - 1) >> shift;
2318 lblock = offset >> shift;
2319 lblock_stop = (offset + len + sdp->sd_sb.sb_bsize - 1) >> shift;
2320 if (lblock_stop > end_of_file && ip != GFS2_I(sdp->sd_rindex))
2323 size = (lblock_stop - lblock) << shift;
2327 gfs2_block_map(&ip->i_inode, lblock, &bh, 0);
2328 if (!buffer_mapped(&bh))
2331 lblock += (bh.b_size >> ip->i_inode.i_blkbits);
2337 static int stuffed_zero_range(struct inode *inode, loff_t offset, loff_t length)
2339 struct gfs2_inode *ip = GFS2_I(inode);
2340 struct buffer_head *dibh;
2343 if (offset >= inode->i_size)
2345 if (offset + length > inode->i_size)
2346 length = inode->i_size - offset;
2348 error = gfs2_meta_inode_buffer(ip, &dibh);
2351 gfs2_trans_add_meta(ip->i_gl, dibh);
2352 memset(dibh->b_data + sizeof(struct gfs2_dinode) + offset, 0,
2358 static int gfs2_journaled_truncate_range(struct inode *inode, loff_t offset,
2361 struct gfs2_sbd *sdp = GFS2_SB(inode);
2362 loff_t max_chunk = GFS2_JTRUNC_REVOKES * sdp->sd_vfs->s_blocksize;
2366 struct gfs2_trans *tr;
2371 if (chunk > max_chunk)
2374 offs = offset & ~PAGE_MASK;
2375 if (offs && chunk > PAGE_SIZE)
2376 chunk = offs + ((chunk - offs) & PAGE_MASK);
2378 truncate_pagecache_range(inode, offset, chunk);
2382 tr = current->journal_info;
2383 if (!test_bit(TR_TOUCHED, &tr->tr_flags))
2386 gfs2_trans_end(sdp);
2387 error = gfs2_trans_begin(sdp, RES_DINODE, GFS2_JTRUNC_REVOKES);
2394 int __gfs2_punch_hole(struct file *file, loff_t offset, loff_t length)
2396 struct inode *inode = file_inode(file);
2397 struct gfs2_inode *ip = GFS2_I(inode);
2398 struct gfs2_sbd *sdp = GFS2_SB(inode);
2399 unsigned int blocksize = i_blocksize(inode);
2403 if (!gfs2_is_stuffed(ip)) {
2404 unsigned int start_off, end_len;
2406 start_off = offset & (blocksize - 1);
2407 end_len = (offset + length) & (blocksize - 1);
2409 unsigned int len = length;
2410 if (length > blocksize - start_off)
2411 len = blocksize - start_off;
2412 error = gfs2_block_zero_range(inode, offset, len);
2415 if (start_off + length < blocksize)
2419 error = gfs2_block_zero_range(inode,
2420 offset + length - end_len, end_len);
2426 start = round_down(offset, blocksize);
2427 end = round_up(offset + length, blocksize) - 1;
2428 error = filemap_write_and_wait_range(inode->i_mapping, start, end);
2432 if (gfs2_is_jdata(ip))
2433 error = gfs2_trans_begin(sdp, RES_DINODE + 2 * RES_JDATA,
2434 GFS2_JTRUNC_REVOKES);
2436 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
2440 if (gfs2_is_stuffed(ip)) {
2441 error = stuffed_zero_range(inode, offset, length);
2446 if (gfs2_is_jdata(ip)) {
2447 BUG_ON(!current->journal_info);
2448 gfs2_journaled_truncate_range(inode, offset, length);
2450 truncate_pagecache_range(inode, offset, offset + length - 1);
2452 file_update_time(file);
2453 mark_inode_dirty(inode);
2455 if (current->journal_info)
2456 gfs2_trans_end(sdp);
2458 if (!gfs2_is_stuffed(ip))
2459 error = punch_hole(ip, offset, length);
2462 if (current->journal_info)
2463 gfs2_trans_end(sdp);
2467 static int gfs2_map_blocks(struct iomap_writepage_ctx *wpc, struct inode *inode,
2472 if (WARN_ON_ONCE(gfs2_is_stuffed(GFS2_I(inode))))
2475 if (offset >= wpc->iomap.offset &&
2476 offset < wpc->iomap.offset + wpc->iomap.length)
2479 memset(&wpc->iomap, 0, sizeof(wpc->iomap));
2480 ret = gfs2_iomap_get(inode, offset, INT_MAX, &wpc->iomap);
2484 const struct iomap_writeback_ops gfs2_writeback_ops = {
2485 .map_blocks = gfs2_map_blocks,