1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
4 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
7 #include <linux/spinlock.h>
8 #include <linux/completion.h>
9 #include <linux/buffer_head.h>
10 #include <linux/blkdev.h>
11 #include <linux/gfs2_ondisk.h>
12 #include <linux/crc32.h>
13 #include <linux/iomap.h>
14 #include <linux/ktime.h>
30 #include "trace_gfs2.h"
32 /* This doesn't need to be that large as max 64 bit pointers in a 4k
33 * block is 512, so __u16 is fine for that. It saves stack space to
37 struct buffer_head *mp_bh[GFS2_MAX_META_HEIGHT];
38 __u16 mp_list[GFS2_MAX_META_HEIGHT];
39 int mp_fheight; /* find_metapath height */
40 int mp_aheight; /* actual height (lookup height) */
43 static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length);
46 * gfs2_unstuffer_page - unstuff a stuffed inode into a block cached by a page
48 * @dibh: the dinode buffer
49 * @block: the block number that was allocated
50 * @page: The (optional) page. This is looked up if @page is NULL
55 static int gfs2_unstuffer_page(struct gfs2_inode *ip, struct buffer_head *dibh,
56 u64 block, struct page *page)
58 struct inode *inode = &ip->i_inode;
59 struct buffer_head *bh;
62 if (!page || page->index) {
63 page = find_or_create_page(inode->i_mapping, 0, GFP_NOFS);
69 if (!PageUptodate(page)) {
70 void *kaddr = kmap(page);
71 u64 dsize = i_size_read(inode);
73 if (dsize > gfs2_max_stuffed_size(ip))
74 dsize = gfs2_max_stuffed_size(ip);
76 memcpy(kaddr, dibh->b_data + sizeof(struct gfs2_dinode), dsize);
77 memset(kaddr + dsize, 0, PAGE_SIZE - dsize);
80 SetPageUptodate(page);
83 if (!page_has_buffers(page))
84 create_empty_buffers(page, BIT(inode->i_blkbits),
87 bh = page_buffers(page);
89 if (!buffer_mapped(bh))
90 map_bh(bh, inode->i_sb, block);
92 set_buffer_uptodate(bh);
93 if (gfs2_is_jdata(ip))
94 gfs2_trans_add_data(ip->i_gl, bh);
96 mark_buffer_dirty(bh);
97 gfs2_ordered_add_inode(ip);
109 * gfs2_unstuff_dinode - Unstuff a dinode when the data has grown too big
110 * @ip: The GFS2 inode to unstuff
111 * @page: The (optional) page. This is looked up if the @page is NULL
113 * This routine unstuffs a dinode and returns it to a "normal" state such
114 * that the height can be grown in the traditional way.
119 int gfs2_unstuff_dinode(struct gfs2_inode *ip, struct page *page)
121 struct buffer_head *bh, *dibh;
122 struct gfs2_dinode *di;
124 int isdir = gfs2_is_dir(ip);
127 down_write(&ip->i_rw_mutex);
129 error = gfs2_meta_inode_buffer(ip, &dibh);
133 if (i_size_read(&ip->i_inode)) {
134 /* Get a free block, fill it with the stuffed data,
135 and write it out to disk */
138 error = gfs2_alloc_blocks(ip, &block, &n, 0, NULL);
142 gfs2_trans_remove_revoke(GFS2_SB(&ip->i_inode), block, 1);
143 error = gfs2_dir_get_new_buffer(ip, block, &bh);
146 gfs2_buffer_copy_tail(bh, sizeof(struct gfs2_meta_header),
147 dibh, sizeof(struct gfs2_dinode));
150 error = gfs2_unstuffer_page(ip, dibh, block, page);
156 /* Set up the pointer to the new block */
158 gfs2_trans_add_meta(ip->i_gl, dibh);
159 di = (struct gfs2_dinode *)dibh->b_data;
160 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
162 if (i_size_read(&ip->i_inode)) {
163 *(__be64 *)(di + 1) = cpu_to_be64(block);
164 gfs2_add_inode_blocks(&ip->i_inode, 1);
165 di->di_blocks = cpu_to_be64(gfs2_get_inode_blocks(&ip->i_inode));
169 di->di_height = cpu_to_be16(1);
174 up_write(&ip->i_rw_mutex);
180 * find_metapath - Find path through the metadata tree
181 * @sdp: The superblock
182 * @block: The disk block to look up
183 * @mp: The metapath to return the result in
184 * @height: The pre-calculated height of the metadata tree
186 * This routine returns a struct metapath structure that defines a path
187 * through the metadata of inode "ip" to get to block "block".
190 * Given: "ip" is a height 3 file, "offset" is 101342453, and this is a
191 * filesystem with a blocksize of 4096.
193 * find_metapath() would return a struct metapath structure set to:
194 * mp_fheight = 3, mp_list[0] = 0, mp_list[1] = 48, and mp_list[2] = 165.
196 * That means that in order to get to the block containing the byte at
197 * offset 101342453, we would load the indirect block pointed to by pointer
198 * 0 in the dinode. We would then load the indirect block pointed to by
199 * pointer 48 in that indirect block. We would then load the data block
200 * pointed to by pointer 165 in that indirect block.
202 * ----------------------------------------
207 * ----------------------------------------
211 * ----------------------------------------
215 * |0 5 6 7 8 9 0 1 2|
216 * ----------------------------------------
220 * ----------------------------------------
225 * ----------------------------------------
229 * ----------------------------------------
230 * | Data block containing offset |
234 * ----------------------------------------
238 static void find_metapath(const struct gfs2_sbd *sdp, u64 block,
239 struct metapath *mp, unsigned int height)
243 mp->mp_fheight = height;
244 for (i = height; i--;)
245 mp->mp_list[i] = do_div(block, sdp->sd_inptrs);
248 static inline unsigned int metapath_branch_start(const struct metapath *mp)
250 if (mp->mp_list[0] == 0)
256 * metaptr1 - Return the first possible metadata pointer in a metapath buffer
257 * @height: The metadata height (0 = dinode)
260 static inline __be64 *metaptr1(unsigned int height, const struct metapath *mp)
262 struct buffer_head *bh = mp->mp_bh[height];
264 return ((__be64 *)(bh->b_data + sizeof(struct gfs2_dinode)));
265 return ((__be64 *)(bh->b_data + sizeof(struct gfs2_meta_header)));
269 * metapointer - Return pointer to start of metadata in a buffer
270 * @height: The metadata height (0 = dinode)
273 * Return a pointer to the block number of the next height of the metadata
274 * tree given a buffer containing the pointer to the current height of the
278 static inline __be64 *metapointer(unsigned int height, const struct metapath *mp)
280 __be64 *p = metaptr1(height, mp);
281 return p + mp->mp_list[height];
284 static inline const __be64 *metaend(unsigned int height, const struct metapath *mp)
286 const struct buffer_head *bh = mp->mp_bh[height];
287 return (const __be64 *)(bh->b_data + bh->b_size);
290 static void clone_metapath(struct metapath *clone, struct metapath *mp)
295 for (hgt = 0; hgt < mp->mp_aheight; hgt++)
296 get_bh(clone->mp_bh[hgt]);
299 static void gfs2_metapath_ra(struct gfs2_glock *gl, __be64 *start, __be64 *end)
303 for (t = start; t < end; t++) {
304 struct buffer_head *rabh;
309 rabh = gfs2_getbuf(gl, be64_to_cpu(*t), CREATE);
310 if (trylock_buffer(rabh)) {
311 if (!buffer_uptodate(rabh)) {
312 rabh->b_end_io = end_buffer_read_sync;
313 submit_bh(REQ_OP_READ,
314 REQ_RAHEAD | REQ_META | REQ_PRIO,
324 static int __fillup_metapath(struct gfs2_inode *ip, struct metapath *mp,
325 unsigned int x, unsigned int h)
328 __be64 *ptr = metapointer(x, mp);
329 u64 dblock = be64_to_cpu(*ptr);
334 ret = gfs2_meta_indirect_buffer(ip, x + 1, dblock, &mp->mp_bh[x + 1]);
338 mp->mp_aheight = x + 1;
343 * lookup_metapath - Walk the metadata tree to a specific point
347 * Assumes that the inode's buffer has already been looked up and
348 * hooked onto mp->mp_bh[0] and that the metapath has been initialised
349 * by find_metapath().
351 * If this function encounters part of the tree which has not been
352 * allocated, it returns the current height of the tree at the point
353 * at which it found the unallocated block. Blocks which are found are
354 * added to the mp->mp_bh[] list.
359 static int lookup_metapath(struct gfs2_inode *ip, struct metapath *mp)
361 return __fillup_metapath(ip, mp, 0, ip->i_height - 1);
365 * fillup_metapath - fill up buffers for the metadata path to a specific height
368 * @h: The height to which it should be mapped
370 * Similar to lookup_metapath, but does lookups for a range of heights
372 * Returns: error or the number of buffers filled
375 static int fillup_metapath(struct gfs2_inode *ip, struct metapath *mp, int h)
381 /* find the first buffer we need to look up. */
382 for (x = h - 1; x > 0; x--) {
387 ret = __fillup_metapath(ip, mp, x, h);
390 return mp->mp_aheight - x - 1;
393 static void release_metapath(struct metapath *mp)
397 for (i = 0; i < GFS2_MAX_META_HEIGHT; i++) {
398 if (mp->mp_bh[i] == NULL)
400 brelse(mp->mp_bh[i]);
406 * gfs2_extent_length - Returns length of an extent of blocks
407 * @bh: The metadata block
408 * @ptr: Current position in @bh
409 * @limit: Max extent length to return
410 * @eob: Set to 1 if we hit "end of block"
412 * Returns: The length of the extent (minimum of one block)
415 static inline unsigned int gfs2_extent_length(struct buffer_head *bh, __be64 *ptr, size_t limit, int *eob)
417 const __be64 *end = (__be64 *)(bh->b_data + bh->b_size);
418 const __be64 *first = ptr;
419 u64 d = be64_to_cpu(*ptr);
427 } while(be64_to_cpu(*ptr) == d);
433 typedef const __be64 *(*gfs2_metadata_walker)(
435 const __be64 *start, const __be64 *end,
436 u64 factor, void *data);
438 #define WALK_STOP ((__be64 *)0)
439 #define WALK_NEXT ((__be64 *)1)
441 static int gfs2_walk_metadata(struct inode *inode, sector_t lblock,
442 u64 len, struct metapath *mp, gfs2_metadata_walker walker,
445 struct metapath clone;
446 struct gfs2_inode *ip = GFS2_I(inode);
447 struct gfs2_sbd *sdp = GFS2_SB(inode);
448 const __be64 *start, *end, *ptr;
453 for (hgt = ip->i_height - 1; hgt >= mp->mp_aheight; hgt--)
454 factor *= sdp->sd_inptrs;
459 /* Walk indirect block. */
460 start = metapointer(hgt, mp);
461 end = metaend(hgt, mp);
463 step = (end - start) * factor;
465 end = start + DIV_ROUND_UP_ULL(len, factor);
467 ptr = walker(mp, start, end, factor, data);
468 if (ptr == WALK_STOP)
473 if (ptr != WALK_NEXT) {
475 mp->mp_list[hgt] += ptr - start;
476 goto fill_up_metapath;
480 /* Decrease height of metapath. */
482 clone_metapath(&clone, mp);
485 brelse(mp->mp_bh[hgt]);
486 mp->mp_bh[hgt] = NULL;
490 factor *= sdp->sd_inptrs;
492 /* Advance in metadata tree. */
493 (mp->mp_list[hgt])++;
494 start = metapointer(hgt, mp);
495 end = metaend(hgt, mp);
497 mp->mp_list[hgt] = 0;
504 /* Increase height of metapath. */
506 clone_metapath(&clone, mp);
509 ret = fillup_metapath(ip, mp, ip->i_height - 1);
514 do_div(factor, sdp->sd_inptrs);
515 mp->mp_aheight = hgt + 1;
518 release_metapath(mp);
522 struct gfs2_hole_walker_args {
526 static const __be64 *gfs2_hole_walker(struct metapath *mp,
527 const __be64 *start, const __be64 *end,
528 u64 factor, void *data)
530 struct gfs2_hole_walker_args *args = data;
533 for (ptr = start; ptr < end; ptr++) {
535 args->blocks += (ptr - start) * factor;
536 if (mp->mp_aheight == mp->mp_fheight)
538 return ptr; /* increase height */
541 args->blocks += (end - start) * factor;
546 * gfs2_hole_size - figure out the size of a hole
548 * @lblock: The logical starting block number
549 * @len: How far to look (in blocks)
550 * @mp: The metapath at lblock
551 * @iomap: The iomap to store the hole size in
553 * This function modifies @mp.
555 * Returns: errno on error
557 static int gfs2_hole_size(struct inode *inode, sector_t lblock, u64 len,
558 struct metapath *mp, struct iomap *iomap)
560 struct gfs2_hole_walker_args args = { };
563 ret = gfs2_walk_metadata(inode, lblock, len, mp, gfs2_hole_walker, &args);
565 iomap->length = args.blocks << inode->i_blkbits;
569 static inline __be64 *gfs2_indirect_init(struct metapath *mp,
570 struct gfs2_glock *gl, unsigned int i,
571 unsigned offset, u64 bn)
573 __be64 *ptr = (__be64 *)(mp->mp_bh[i - 1]->b_data +
574 ((i > 1) ? sizeof(struct gfs2_meta_header) :
575 sizeof(struct gfs2_dinode)));
577 BUG_ON(mp->mp_bh[i] != NULL);
578 mp->mp_bh[i] = gfs2_meta_new(gl, bn);
579 gfs2_trans_add_meta(gl, mp->mp_bh[i]);
580 gfs2_metatype_set(mp->mp_bh[i], GFS2_METATYPE_IN, GFS2_FORMAT_IN);
581 gfs2_buffer_clear_tail(mp->mp_bh[i], sizeof(struct gfs2_meta_header));
583 *ptr = cpu_to_be64(bn);
589 ALLOC_GROW_DEPTH = 1,
590 ALLOC_GROW_HEIGHT = 2,
591 /* ALLOC_UNSTUFF = 3, TBD and rather complicated */
595 * gfs2_iomap_alloc - Build a metadata tree of the requested height
596 * @inode: The GFS2 inode
597 * @iomap: The iomap structure
598 * @flags: iomap flags
599 * @mp: The metapath, with proper height information calculated
601 * In this routine we may have to alloc:
602 * i) Indirect blocks to grow the metadata tree height
603 * ii) Indirect blocks to fill in lower part of the metadata tree
606 * This function is called after gfs2_iomap_get, which works out the
607 * total number of blocks which we need via gfs2_alloc_size.
609 * We then do the actual allocation asking for an extent at a time (if
610 * enough contiguous free blocks are available, there will only be one
611 * allocation request per call) and uses the state machine to initialise
612 * the blocks in order.
614 * Right now, this function will allocate at most one indirect block
615 * worth of data -- with a default block size of 4K, that's slightly
616 * less than 2M. If this limitation is ever removed to allow huge
617 * allocations, we would probably still want to limit the iomap size we
618 * return to avoid stalling other tasks during huge writes; the next
619 * iomap iteration would then find the blocks already allocated.
621 * Returns: errno on error
624 static int gfs2_iomap_alloc(struct inode *inode, struct iomap *iomap,
625 unsigned flags, struct metapath *mp)
627 struct gfs2_inode *ip = GFS2_I(inode);
628 struct gfs2_sbd *sdp = GFS2_SB(inode);
629 struct buffer_head *dibh = mp->mp_bh[0];
631 unsigned n, i, blks, alloced = 0, iblks = 0, branch_start = 0;
632 size_t dblks = iomap->length >> inode->i_blkbits;
633 const unsigned end_of_metadata = mp->mp_fheight - 1;
635 enum alloc_state state;
639 BUG_ON(mp->mp_aheight < 1);
640 BUG_ON(dibh == NULL);
643 gfs2_trans_add_meta(ip->i_gl, dibh);
645 down_write(&ip->i_rw_mutex);
647 if (mp->mp_fheight == mp->mp_aheight) {
648 /* Bottom indirect block exists */
651 /* Need to allocate indirect blocks */
652 if (mp->mp_fheight == ip->i_height) {
653 /* Writing into existing tree, extend tree down */
654 iblks = mp->mp_fheight - mp->mp_aheight;
655 state = ALLOC_GROW_DEPTH;
657 /* Building up tree height */
658 state = ALLOC_GROW_HEIGHT;
659 iblks = mp->mp_fheight - ip->i_height;
660 branch_start = metapath_branch_start(mp);
661 iblks += (mp->mp_fheight - branch_start);
665 /* start of the second part of the function (state machine) */
667 blks = dblks + iblks;
671 ret = gfs2_alloc_blocks(ip, &bn, &n, 0, NULL);
675 if (state != ALLOC_DATA || gfs2_is_jdata(ip))
676 gfs2_trans_remove_revoke(sdp, bn, n);
678 /* Growing height of tree */
679 case ALLOC_GROW_HEIGHT:
681 ptr = (__be64 *)(dibh->b_data +
682 sizeof(struct gfs2_dinode));
685 for (; i - 1 < mp->mp_fheight - ip->i_height && n > 0;
687 gfs2_indirect_init(mp, ip->i_gl, i, 0, bn++);
688 if (i - 1 == mp->mp_fheight - ip->i_height) {
690 gfs2_buffer_copy_tail(mp->mp_bh[i],
691 sizeof(struct gfs2_meta_header),
692 dibh, sizeof(struct gfs2_dinode));
693 gfs2_buffer_clear_tail(dibh,
694 sizeof(struct gfs2_dinode) +
696 ptr = (__be64 *)(mp->mp_bh[i]->b_data +
697 sizeof(struct gfs2_meta_header));
699 state = ALLOC_GROW_DEPTH;
700 for(i = branch_start; i < mp->mp_fheight; i++) {
701 if (mp->mp_bh[i] == NULL)
703 brelse(mp->mp_bh[i]);
710 /* fall through - To branching from existing tree */
711 case ALLOC_GROW_DEPTH:
712 if (i > 1 && i < mp->mp_fheight)
713 gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[i-1]);
714 for (; i < mp->mp_fheight && n > 0; i++, n--)
715 gfs2_indirect_init(mp, ip->i_gl, i,
716 mp->mp_list[i-1], bn++);
717 if (i == mp->mp_fheight)
721 /* fall through - To tree complete, adding data blocks */
724 BUG_ON(mp->mp_bh[end_of_metadata] == NULL);
725 gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[end_of_metadata]);
727 ptr = metapointer(end_of_metadata, mp);
728 iomap->addr = bn << inode->i_blkbits;
729 iomap->flags |= IOMAP_F_MERGED | IOMAP_F_NEW;
731 *ptr++ = cpu_to_be64(bn++);
734 } while (iomap->addr == IOMAP_NULL_ADDR);
736 iomap->type = IOMAP_MAPPED;
737 iomap->length = (u64)dblks << inode->i_blkbits;
738 ip->i_height = mp->mp_fheight;
739 gfs2_add_inode_blocks(&ip->i_inode, alloced);
740 gfs2_dinode_out(ip, dibh->b_data);
742 up_write(&ip->i_rw_mutex);
746 #define IOMAP_F_GFS2_BOUNDARY IOMAP_F_PRIVATE
749 * gfs2_alloc_size - Compute the maximum allocation size
752 * @size: Requested size in blocks
754 * Compute the maximum size of the next allocation at @mp.
756 * Returns: size in blocks
758 static u64 gfs2_alloc_size(struct inode *inode, struct metapath *mp, u64 size)
760 struct gfs2_inode *ip = GFS2_I(inode);
761 struct gfs2_sbd *sdp = GFS2_SB(inode);
762 const __be64 *first, *ptr, *end;
765 * For writes to stuffed files, this function is called twice via
766 * gfs2_iomap_get, before and after unstuffing. The size we return the
767 * first time needs to be large enough to get the reservation and
768 * allocation sizes right. The size we return the second time must
769 * be exact or else gfs2_iomap_alloc won't do the right thing.
772 if (gfs2_is_stuffed(ip) || mp->mp_fheight != mp->mp_aheight) {
773 unsigned int maxsize = mp->mp_fheight > 1 ?
774 sdp->sd_inptrs : sdp->sd_diptrs;
775 maxsize -= mp->mp_list[mp->mp_fheight - 1];
781 first = metapointer(ip->i_height - 1, mp);
782 end = metaend(ip->i_height - 1, mp);
783 if (end - first > size)
785 for (ptr = first; ptr < end; ptr++) {
793 * gfs2_iomap_get - Map blocks from an inode to disk blocks
795 * @pos: Starting position in bytes
796 * @length: Length to map, in bytes
797 * @flags: iomap flags
798 * @iomap: The iomap structure
803 static int gfs2_iomap_get(struct inode *inode, loff_t pos, loff_t length,
804 unsigned flags, struct iomap *iomap,
807 struct gfs2_inode *ip = GFS2_I(inode);
808 struct gfs2_sbd *sdp = GFS2_SB(inode);
809 loff_t size = i_size_read(inode);
812 sector_t lblock_stop;
816 struct buffer_head *dibh = NULL, *bh;
822 down_read(&ip->i_rw_mutex);
824 ret = gfs2_meta_inode_buffer(ip, &dibh);
829 if (gfs2_is_stuffed(ip)) {
830 if (flags & IOMAP_WRITE) {
831 loff_t max_size = gfs2_max_stuffed_size(ip);
833 if (pos + length > max_size)
835 iomap->length = max_size;
838 if (flags & IOMAP_REPORT) {
844 iomap->length = length;
848 iomap->length = size;
850 iomap->addr = (ip->i_no_addr << inode->i_blkbits) +
851 sizeof(struct gfs2_dinode);
852 iomap->type = IOMAP_INLINE;
853 iomap->inline_data = dibh->b_data + sizeof(struct gfs2_dinode);
858 lblock = pos >> inode->i_blkbits;
859 iomap->offset = lblock << inode->i_blkbits;
860 lblock_stop = (pos + length - 1) >> inode->i_blkbits;
861 len = lblock_stop - lblock + 1;
862 iomap->length = len << inode->i_blkbits;
864 height = ip->i_height;
865 while ((lblock + 1) * sdp->sd_sb.sb_bsize > sdp->sd_heightsize[height])
867 find_metapath(sdp, lblock, mp, height);
868 if (height > ip->i_height || gfs2_is_stuffed(ip))
871 ret = lookup_metapath(ip, mp);
875 if (mp->mp_aheight != ip->i_height)
878 ptr = metapointer(ip->i_height - 1, mp);
882 bh = mp->mp_bh[ip->i_height - 1];
883 len = gfs2_extent_length(bh, ptr, len, &eob);
885 iomap->addr = be64_to_cpu(*ptr) << inode->i_blkbits;
886 iomap->length = len << inode->i_blkbits;
887 iomap->type = IOMAP_MAPPED;
888 iomap->flags |= IOMAP_F_MERGED;
890 iomap->flags |= IOMAP_F_GFS2_BOUNDARY;
893 iomap->bdev = inode->i_sb->s_bdev;
895 up_read(&ip->i_rw_mutex);
899 iomap->addr = IOMAP_NULL_ADDR;
900 iomap->type = IOMAP_HOLE;
901 if (flags & IOMAP_REPORT) {
904 else if (height == ip->i_height)
905 ret = gfs2_hole_size(inode, lblock, len, mp, iomap);
907 iomap->length = size - pos;
908 } else if (flags & IOMAP_WRITE) {
911 if (flags & IOMAP_DIRECT)
912 goto out; /* (see gfs2_file_direct_write) */
914 len = gfs2_alloc_size(inode, mp, len);
915 alloc_size = len << inode->i_blkbits;
916 if (alloc_size < iomap->length)
917 iomap->length = alloc_size;
919 if (pos < size && height == ip->i_height)
920 ret = gfs2_hole_size(inode, lblock, len, mp, iomap);
926 * gfs2_lblk_to_dblk - convert logical block to disk block
927 * @inode: the inode of the file we're mapping
928 * @lblock: the block relative to the start of the file
929 * @dblock: the returned dblock, if no error
931 * This function maps a single block from a file logical block (relative to
932 * the start of the file) to a file system absolute block using iomap.
934 * Returns: the absolute file system block, or an error
936 int gfs2_lblk_to_dblk(struct inode *inode, u32 lblock, u64 *dblock)
938 struct iomap iomap = { };
939 struct metapath mp = { .mp_aheight = 1, };
940 loff_t pos = (loff_t)lblock << inode->i_blkbits;
943 ret = gfs2_iomap_get(inode, pos, i_blocksize(inode), 0, &iomap, &mp);
944 release_metapath(&mp);
946 *dblock = iomap.addr >> inode->i_blkbits;
951 static int gfs2_write_lock(struct inode *inode)
953 struct gfs2_inode *ip = GFS2_I(inode);
954 struct gfs2_sbd *sdp = GFS2_SB(inode);
957 gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &ip->i_gh);
958 error = gfs2_glock_nq(&ip->i_gh);
961 if (&ip->i_inode == sdp->sd_rindex) {
962 struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
964 error = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE,
965 GL_NOCACHE, &m_ip->i_gh);
972 gfs2_glock_dq(&ip->i_gh);
974 gfs2_holder_uninit(&ip->i_gh);
978 static void gfs2_write_unlock(struct inode *inode)
980 struct gfs2_inode *ip = GFS2_I(inode);
981 struct gfs2_sbd *sdp = GFS2_SB(inode);
983 if (&ip->i_inode == sdp->sd_rindex) {
984 struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
986 gfs2_glock_dq_uninit(&m_ip->i_gh);
988 gfs2_glock_dq_uninit(&ip->i_gh);
991 static int gfs2_iomap_page_prepare(struct inode *inode, loff_t pos,
992 unsigned len, struct iomap *iomap)
994 unsigned int blockmask = i_blocksize(inode) - 1;
995 struct gfs2_sbd *sdp = GFS2_SB(inode);
998 blocks = ((pos & blockmask) + len + blockmask) >> inode->i_blkbits;
999 return gfs2_trans_begin(sdp, RES_DINODE + blocks, 0);
1002 static void gfs2_iomap_page_done(struct inode *inode, loff_t pos,
1003 unsigned copied, struct page *page,
1004 struct iomap *iomap)
1006 struct gfs2_inode *ip = GFS2_I(inode);
1007 struct gfs2_sbd *sdp = GFS2_SB(inode);
1009 if (page && !gfs2_is_stuffed(ip))
1010 gfs2_page_add_databufs(ip, page, offset_in_page(pos), copied);
1011 gfs2_trans_end(sdp);
1014 static const struct iomap_page_ops gfs2_iomap_page_ops = {
1015 .page_prepare = gfs2_iomap_page_prepare,
1016 .page_done = gfs2_iomap_page_done,
1019 static int gfs2_iomap_begin_write(struct inode *inode, loff_t pos,
1020 loff_t length, unsigned flags,
1021 struct iomap *iomap,
1022 struct metapath *mp)
1024 struct gfs2_inode *ip = GFS2_I(inode);
1025 struct gfs2_sbd *sdp = GFS2_SB(inode);
1026 unsigned int data_blocks = 0, ind_blocks = 0, rblocks;
1027 bool unstuff, alloc_required;
1030 ret = gfs2_write_lock(inode);
1034 unstuff = gfs2_is_stuffed(ip) &&
1035 pos + length > gfs2_max_stuffed_size(ip);
1037 ret = gfs2_iomap_get(inode, pos, length, flags, iomap, mp);
1041 alloc_required = unstuff || iomap->type == IOMAP_HOLE;
1043 if (alloc_required || gfs2_is_jdata(ip))
1044 gfs2_write_calc_reserv(ip, iomap->length, &data_blocks,
1047 if (alloc_required) {
1048 struct gfs2_alloc_parms ap = {
1049 .target = data_blocks + ind_blocks
1052 ret = gfs2_quota_lock_check(ip, &ap);
1056 ret = gfs2_inplace_reserve(ip, &ap);
1061 rblocks = RES_DINODE + ind_blocks;
1062 if (gfs2_is_jdata(ip))
1063 rblocks += data_blocks;
1064 if (ind_blocks || data_blocks)
1065 rblocks += RES_STATFS + RES_QUOTA;
1066 if (inode == sdp->sd_rindex)
1067 rblocks += 2 * RES_STATFS;
1069 rblocks += gfs2_rg_blocks(ip, data_blocks + ind_blocks);
1071 if (unstuff || iomap->type == IOMAP_HOLE) {
1072 struct gfs2_trans *tr;
1074 ret = gfs2_trans_begin(sdp, rblocks,
1075 iomap->length >> inode->i_blkbits);
1077 goto out_trans_fail;
1080 ret = gfs2_unstuff_dinode(ip, NULL);
1083 release_metapath(mp);
1084 ret = gfs2_iomap_get(inode, iomap->offset,
1085 iomap->length, flags, iomap, mp);
1090 if (iomap->type == IOMAP_HOLE) {
1091 ret = gfs2_iomap_alloc(inode, iomap, flags, mp);
1093 gfs2_trans_end(sdp);
1094 gfs2_inplace_release(ip);
1095 punch_hole(ip, iomap->offset, iomap->length);
1100 tr = current->journal_info;
1101 if (tr->tr_num_buf_new)
1102 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1104 gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[0]);
1106 gfs2_trans_end(sdp);
1109 if (gfs2_is_stuffed(ip) || gfs2_is_jdata(ip))
1110 iomap->page_ops = &gfs2_iomap_page_ops;
1114 gfs2_trans_end(sdp);
1117 gfs2_inplace_release(ip);
1120 gfs2_quota_unlock(ip);
1122 gfs2_write_unlock(inode);
1126 static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
1127 unsigned flags, struct iomap *iomap)
1129 struct gfs2_inode *ip = GFS2_I(inode);
1130 struct metapath mp = { .mp_aheight = 1, };
1133 iomap->flags |= IOMAP_F_BUFFER_HEAD;
1135 trace_gfs2_iomap_start(ip, pos, length, flags);
1136 if ((flags & IOMAP_WRITE) && !(flags & IOMAP_DIRECT)) {
1137 ret = gfs2_iomap_begin_write(inode, pos, length, flags, iomap, &mp);
1139 ret = gfs2_iomap_get(inode, pos, length, flags, iomap, &mp);
1142 * Silently fall back to buffered I/O for stuffed files or if
1143 * we've hot a hole (see gfs2_file_direct_write).
1145 if ((flags & IOMAP_WRITE) && (flags & IOMAP_DIRECT) &&
1146 iomap->type != IOMAP_MAPPED)
1149 release_metapath(&mp);
1150 trace_gfs2_iomap_end(ip, iomap, ret);
1154 static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length,
1155 ssize_t written, unsigned flags, struct iomap *iomap)
1157 struct gfs2_inode *ip = GFS2_I(inode);
1158 struct gfs2_sbd *sdp = GFS2_SB(inode);
1160 if ((flags & (IOMAP_WRITE | IOMAP_DIRECT)) != IOMAP_WRITE)
1163 if (!gfs2_is_stuffed(ip))
1164 gfs2_ordered_add_inode(ip);
1166 if (inode == sdp->sd_rindex)
1167 adjust_fs_space(inode);
1169 gfs2_inplace_release(ip);
1171 if (length != written && (iomap->flags & IOMAP_F_NEW)) {
1172 /* Deallocate blocks that were just allocated. */
1173 loff_t blockmask = i_blocksize(inode) - 1;
1174 loff_t end = (pos + length) & ~blockmask;
1176 pos = (pos + written + blockmask) & ~blockmask;
1178 truncate_pagecache_range(inode, pos, end - 1);
1179 punch_hole(ip, pos, end - pos);
1183 if (ip->i_qadata && ip->i_qadata->qa_qd_num)
1184 gfs2_quota_unlock(ip);
1185 if (iomap->flags & IOMAP_F_SIZE_CHANGED)
1186 mark_inode_dirty(inode);
1187 gfs2_write_unlock(inode);
1193 const struct iomap_ops gfs2_iomap_ops = {
1194 .iomap_begin = gfs2_iomap_begin,
1195 .iomap_end = gfs2_iomap_end,
1199 * gfs2_block_map - Map one or more blocks of an inode to a disk block
1201 * @lblock: The logical block number
1202 * @bh_map: The bh to be mapped
1203 * @create: True if its ok to alloc blocks to satify the request
1205 * The size of the requested mapping is defined in bh_map->b_size.
1207 * Clears buffer_mapped(bh_map) and leaves bh_map->b_size unchanged
1208 * when @lblock is not mapped. Sets buffer_mapped(bh_map) and
1209 * bh_map->b_size to indicate the size of the mapping when @lblock and
1210 * successive blocks are mapped, up to the requested size.
1212 * Sets buffer_boundary() if a read of metadata will be required
1213 * before the next block can be mapped. Sets buffer_new() if new
1214 * blocks were allocated.
1219 int gfs2_block_map(struct inode *inode, sector_t lblock,
1220 struct buffer_head *bh_map, int create)
1222 struct gfs2_inode *ip = GFS2_I(inode);
1223 loff_t pos = (loff_t)lblock << inode->i_blkbits;
1224 loff_t length = bh_map->b_size;
1225 struct metapath mp = { .mp_aheight = 1, };
1226 struct iomap iomap = { };
1229 clear_buffer_mapped(bh_map);
1230 clear_buffer_new(bh_map);
1231 clear_buffer_boundary(bh_map);
1232 trace_gfs2_bmap(ip, bh_map, lblock, create, 1);
1235 ret = gfs2_iomap_get(inode, pos, length, IOMAP_WRITE, &iomap, &mp);
1236 if (!ret && iomap.type == IOMAP_HOLE)
1237 ret = gfs2_iomap_alloc(inode, &iomap, IOMAP_WRITE, &mp);
1238 release_metapath(&mp);
1240 ret = gfs2_iomap_get(inode, pos, length, 0, &iomap, &mp);
1241 release_metapath(&mp);
1246 if (iomap.length > bh_map->b_size) {
1247 iomap.length = bh_map->b_size;
1248 iomap.flags &= ~IOMAP_F_GFS2_BOUNDARY;
1250 if (iomap.addr != IOMAP_NULL_ADDR)
1251 map_bh(bh_map, inode->i_sb, iomap.addr >> inode->i_blkbits);
1252 bh_map->b_size = iomap.length;
1253 if (iomap.flags & IOMAP_F_GFS2_BOUNDARY)
1254 set_buffer_boundary(bh_map);
1255 if (iomap.flags & IOMAP_F_NEW)
1256 set_buffer_new(bh_map);
1259 trace_gfs2_bmap(ip, bh_map, lblock, create, ret);
1264 * Deprecated: do not use in new code
1266 int gfs2_extent_map(struct inode *inode, u64 lblock, int *new, u64 *dblock, unsigned *extlen)
1268 struct buffer_head bh = { .b_state = 0, .b_blocknr = 0 };
1276 bh.b_size = BIT(inode->i_blkbits + (create ? 0 : 5));
1277 ret = gfs2_block_map(inode, lblock, &bh, create);
1278 *extlen = bh.b_size >> inode->i_blkbits;
1279 *dblock = bh.b_blocknr;
1280 if (buffer_new(&bh))
1288 * gfs2_block_zero_range - Deal with zeroing out data
1290 * This is partly borrowed from ext3.
1292 static int gfs2_block_zero_range(struct inode *inode, loff_t from,
1293 unsigned int length)
1295 struct address_space *mapping = inode->i_mapping;
1296 struct gfs2_inode *ip = GFS2_I(inode);
1297 unsigned long index = from >> PAGE_SHIFT;
1298 unsigned offset = from & (PAGE_SIZE-1);
1299 unsigned blocksize, iblock, pos;
1300 struct buffer_head *bh;
1304 page = find_or_create_page(mapping, index, GFP_NOFS);
1308 blocksize = inode->i_sb->s_blocksize;
1309 iblock = index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits);
1311 if (!page_has_buffers(page))
1312 create_empty_buffers(page, blocksize, 0);
1314 /* Find the buffer that contains "offset" */
1315 bh = page_buffers(page);
1317 while (offset >= pos) {
1318 bh = bh->b_this_page;
1325 if (!buffer_mapped(bh)) {
1326 gfs2_block_map(inode, iblock, bh, 0);
1327 /* unmapped? It's a hole - nothing to do */
1328 if (!buffer_mapped(bh))
1332 /* Ok, it's mapped. Make sure it's up-to-date */
1333 if (PageUptodate(page))
1334 set_buffer_uptodate(bh);
1336 if (!buffer_uptodate(bh)) {
1338 ll_rw_block(REQ_OP_READ, 0, 1, &bh);
1340 /* Uhhuh. Read error. Complain and punt. */
1341 if (!buffer_uptodate(bh))
1346 if (gfs2_is_jdata(ip))
1347 gfs2_trans_add_data(ip->i_gl, bh);
1349 gfs2_ordered_add_inode(ip);
1351 zero_user(page, offset, length);
1352 mark_buffer_dirty(bh);
1359 #define GFS2_JTRUNC_REVOKES 8192
1362 * gfs2_journaled_truncate - Wrapper for truncate_pagecache for jdata files
1363 * @inode: The inode being truncated
1364 * @oldsize: The original (larger) size
1365 * @newsize: The new smaller size
1367 * With jdata files, we have to journal a revoke for each block which is
1368 * truncated. As a result, we need to split this into separate transactions
1369 * if the number of pages being truncated gets too large.
1372 static int gfs2_journaled_truncate(struct inode *inode, u64 oldsize, u64 newsize)
1374 struct gfs2_sbd *sdp = GFS2_SB(inode);
1375 u64 max_chunk = GFS2_JTRUNC_REVOKES * sdp->sd_vfs->s_blocksize;
1379 while (oldsize != newsize) {
1380 struct gfs2_trans *tr;
1383 chunk = oldsize - newsize;
1384 if (chunk > max_chunk)
1387 offs = oldsize & ~PAGE_MASK;
1388 if (offs && chunk > PAGE_SIZE)
1389 chunk = offs + ((chunk - offs) & PAGE_MASK);
1391 truncate_pagecache(inode, oldsize - chunk);
1394 tr = current->journal_info;
1395 if (!test_bit(TR_TOUCHED, &tr->tr_flags))
1398 gfs2_trans_end(sdp);
1399 error = gfs2_trans_begin(sdp, RES_DINODE, GFS2_JTRUNC_REVOKES);
1407 static int trunc_start(struct inode *inode, u64 newsize)
1409 struct gfs2_inode *ip = GFS2_I(inode);
1410 struct gfs2_sbd *sdp = GFS2_SB(inode);
1411 struct buffer_head *dibh = NULL;
1412 int journaled = gfs2_is_jdata(ip);
1413 u64 oldsize = inode->i_size;
1417 error = gfs2_trans_begin(sdp, RES_DINODE + RES_JDATA, GFS2_JTRUNC_REVOKES);
1419 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
1423 error = gfs2_meta_inode_buffer(ip, &dibh);
1427 gfs2_trans_add_meta(ip->i_gl, dibh);
1429 if (gfs2_is_stuffed(ip)) {
1430 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode) + newsize);
1432 unsigned int blocksize = i_blocksize(inode);
1433 unsigned int offs = newsize & (blocksize - 1);
1435 error = gfs2_block_zero_range(inode, newsize,
1440 ip->i_diskflags |= GFS2_DIF_TRUNC_IN_PROG;
1443 i_size_write(inode, newsize);
1444 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
1445 gfs2_dinode_out(ip, dibh->b_data);
1448 error = gfs2_journaled_truncate(inode, oldsize, newsize);
1450 truncate_pagecache(inode, newsize);
1454 if (current->journal_info)
1455 gfs2_trans_end(sdp);
1459 int gfs2_iomap_get_alloc(struct inode *inode, loff_t pos, loff_t length,
1460 struct iomap *iomap)
1462 struct metapath mp = { .mp_aheight = 1, };
1465 ret = gfs2_iomap_get(inode, pos, length, IOMAP_WRITE, iomap, &mp);
1466 if (!ret && iomap->type == IOMAP_HOLE)
1467 ret = gfs2_iomap_alloc(inode, iomap, IOMAP_WRITE, &mp);
1468 release_metapath(&mp);
1473 * sweep_bh_for_rgrps - find an rgrp in a meta buffer and free blocks therein
1475 * @rg_gh: holder of resource group glock
1476 * @bh: buffer head to sweep
1477 * @start: starting point in bh
1478 * @end: end point in bh
1479 * @meta: true if bh points to metadata (rather than data)
1480 * @btotal: place to keep count of total blocks freed
1482 * We sweep a metadata buffer (provided by the metapath) for blocks we need to
1483 * free, and free them all. However, we do it one rgrp at a time. If this
1484 * block has references to multiple rgrps, we break it into individual
1485 * transactions. This allows other processes to use the rgrps while we're
1486 * focused on a single one, for better concurrency / performance.
1487 * At every transaction boundary, we rewrite the inode into the journal.
1488 * That way the bitmaps are kept consistent with the inode and we can recover
1489 * if we're interrupted by power-outages.
1491 * Returns: 0, or return code if an error occurred.
1492 * *btotal has the total number of blocks freed
1494 static int sweep_bh_for_rgrps(struct gfs2_inode *ip, struct gfs2_holder *rd_gh,
1495 struct buffer_head *bh, __be64 *start, __be64 *end,
1496 bool meta, u32 *btotal)
1498 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1499 struct gfs2_rgrpd *rgd;
1500 struct gfs2_trans *tr;
1502 int blks_outside_rgrp;
1503 u64 bn, bstart, isize_blks;
1504 s64 blen; /* needs to be s64 or gfs2_add_inode_blocks breaks */
1506 bool buf_in_tr = false; /* buffer was added to transaction */
1510 if (gfs2_holder_initialized(rd_gh)) {
1511 rgd = gfs2_glock2rgrp(rd_gh->gh_gl);
1512 gfs2_assert_withdraw(sdp,
1513 gfs2_glock_is_locked_by_me(rd_gh->gh_gl));
1515 blks_outside_rgrp = 0;
1519 for (p = start; p < end; p++) {
1522 bn = be64_to_cpu(*p);
1525 if (!rgrp_contains_block(rgd, bn)) {
1526 blks_outside_rgrp++;
1530 rgd = gfs2_blk2rgrpd(sdp, bn, true);
1531 if (unlikely(!rgd)) {
1535 ret = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE,
1540 /* Must be done with the rgrp glock held: */
1541 if (gfs2_rs_active(&ip->i_res) &&
1542 rgd == ip->i_res.rs_rbm.rgd)
1543 gfs2_rs_deltree(&ip->i_res);
1546 /* The size of our transactions will be unknown until we
1547 actually process all the metadata blocks that relate to
1548 the rgrp. So we estimate. We know it can't be more than
1549 the dinode's i_blocks and we don't want to exceed the
1550 journal flush threshold, sd_log_thresh2. */
1551 if (current->journal_info == NULL) {
1552 unsigned int jblocks_rqsted, revokes;
1554 jblocks_rqsted = rgd->rd_length + RES_DINODE +
1556 isize_blks = gfs2_get_inode_blocks(&ip->i_inode);
1557 if (isize_blks > atomic_read(&sdp->sd_log_thresh2))
1559 atomic_read(&sdp->sd_log_thresh2);
1561 jblocks_rqsted += isize_blks;
1562 revokes = jblocks_rqsted;
1564 revokes += end - start;
1565 else if (ip->i_depth)
1566 revokes += sdp->sd_inptrs;
1567 ret = gfs2_trans_begin(sdp, jblocks_rqsted, revokes);
1570 down_write(&ip->i_rw_mutex);
1572 /* check if we will exceed the transaction blocks requested */
1573 tr = current->journal_info;
1574 if (tr->tr_num_buf_new + RES_STATFS +
1575 RES_QUOTA >= atomic_read(&sdp->sd_log_thresh2)) {
1576 /* We set blks_outside_rgrp to ensure the loop will
1577 be repeated for the same rgrp, but with a new
1579 blks_outside_rgrp++;
1580 /* This next part is tricky. If the buffer was added
1581 to the transaction, we've already set some block
1582 pointers to 0, so we better follow through and free
1583 them, or we will introduce corruption (so break).
1584 This may be impossible, or at least rare, but I
1585 decided to cover the case regardless.
1587 If the buffer was not added to the transaction
1588 (this call), doing so would exceed our transaction
1589 size, so we need to end the transaction and start a
1590 new one (so goto). */
1597 gfs2_trans_add_meta(ip->i_gl, bh);
1600 if (bstart + blen == bn) {
1605 __gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta);
1607 gfs2_add_inode_blocks(&ip->i_inode, -blen);
1613 __gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta);
1615 gfs2_add_inode_blocks(&ip->i_inode, -blen);
1618 if (!ret && blks_outside_rgrp) { /* If buffer still has non-zero blocks
1619 outside the rgrp we just processed,
1620 do it all over again. */
1621 if (current->journal_info) {
1622 struct buffer_head *dibh;
1624 ret = gfs2_meta_inode_buffer(ip, &dibh);
1628 /* Every transaction boundary, we rewrite the dinode
1629 to keep its di_blocks current in case of failure. */
1630 ip->i_inode.i_mtime = ip->i_inode.i_ctime =
1631 current_time(&ip->i_inode);
1632 gfs2_trans_add_meta(ip->i_gl, dibh);
1633 gfs2_dinode_out(ip, dibh->b_data);
1635 up_write(&ip->i_rw_mutex);
1636 gfs2_trans_end(sdp);
1638 gfs2_glock_dq_uninit(rd_gh);
1646 static bool mp_eq_to_hgt(struct metapath *mp, __u16 *list, unsigned int h)
1648 if (memcmp(mp->mp_list, list, h * sizeof(mp->mp_list[0])))
1654 * find_nonnull_ptr - find a non-null pointer given a metapath and height
1655 * @mp: starting metapath
1656 * @h: desired height to search
1658 * Assumes the metapath is valid (with buffers) out to height h.
1659 * Returns: true if a non-null pointer was found in the metapath buffer
1660 * false if all remaining pointers are NULL in the buffer
1662 static bool find_nonnull_ptr(struct gfs2_sbd *sdp, struct metapath *mp,
1664 __u16 *end_list, unsigned int end_aligned)
1666 struct buffer_head *bh = mp->mp_bh[h];
1667 __be64 *first, *ptr, *end;
1669 first = metaptr1(h, mp);
1670 ptr = first + mp->mp_list[h];
1671 end = (__be64 *)(bh->b_data + bh->b_size);
1672 if (end_list && mp_eq_to_hgt(mp, end_list, h)) {
1673 bool keep_end = h < end_aligned;
1674 end = first + end_list[h] + keep_end;
1678 if (*ptr) { /* if we have a non-null pointer */
1679 mp->mp_list[h] = ptr - first;
1681 if (h < GFS2_MAX_META_HEIGHT)
1690 enum dealloc_states {
1691 DEALLOC_MP_FULL = 0, /* Strip a metapath with all buffers read in */
1692 DEALLOC_MP_LOWER = 1, /* lower the metapath strip height */
1693 DEALLOC_FILL_MP = 2, /* Fill in the metapath to the given height. */
1694 DEALLOC_DONE = 3, /* process complete */
1698 metapointer_range(struct metapath *mp, int height,
1699 __u16 *start_list, unsigned int start_aligned,
1700 __u16 *end_list, unsigned int end_aligned,
1701 __be64 **start, __be64 **end)
1703 struct buffer_head *bh = mp->mp_bh[height];
1706 first = metaptr1(height, mp);
1708 if (mp_eq_to_hgt(mp, start_list, height)) {
1709 bool keep_start = height < start_aligned;
1710 *start = first + start_list[height] + keep_start;
1712 *end = (__be64 *)(bh->b_data + bh->b_size);
1713 if (end_list && mp_eq_to_hgt(mp, end_list, height)) {
1714 bool keep_end = height < end_aligned;
1715 *end = first + end_list[height] + keep_end;
1719 static inline bool walk_done(struct gfs2_sbd *sdp,
1720 struct metapath *mp, int height,
1721 __u16 *end_list, unsigned int end_aligned)
1726 bool keep_end = height < end_aligned;
1727 if (!mp_eq_to_hgt(mp, end_list, height))
1729 end = end_list[height] + keep_end;
1731 end = (height > 0) ? sdp->sd_inptrs : sdp->sd_diptrs;
1732 return mp->mp_list[height] >= end;
1736 * punch_hole - deallocate blocks in a file
1737 * @ip: inode to truncate
1738 * @offset: the start of the hole
1739 * @length: the size of the hole (or 0 for truncate)
1741 * Punch a hole into a file or truncate a file at a given position. This
1742 * function operates in whole blocks (@offset and @length are rounded
1743 * accordingly); partially filled blocks must be cleared otherwise.
1745 * This function works from the bottom up, and from the right to the left. In
1746 * other words, it strips off the highest layer (data) before stripping any of
1747 * the metadata. Doing it this way is best in case the operation is interrupted
1748 * by power failure, etc. The dinode is rewritten in every transaction to
1749 * guarantee integrity.
1751 static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length)
1753 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1754 u64 maxsize = sdp->sd_heightsize[ip->i_height];
1755 struct metapath mp = {};
1756 struct buffer_head *dibh, *bh;
1757 struct gfs2_holder rd_gh;
1758 unsigned int bsize_shift = sdp->sd_sb.sb_bsize_shift;
1759 u64 lblock = (offset + (1 << bsize_shift) - 1) >> bsize_shift;
1760 __u16 start_list[GFS2_MAX_META_HEIGHT];
1761 __u16 __end_list[GFS2_MAX_META_HEIGHT], *end_list = NULL;
1762 unsigned int start_aligned, uninitialized_var(end_aligned);
1763 unsigned int strip_h = ip->i_height - 1;
1766 int mp_h; /* metapath buffers are read in to this height */
1768 __be64 *start, *end;
1770 if (offset >= maxsize) {
1772 * The starting point lies beyond the allocated meta-data;
1773 * there are no blocks do deallocate.
1779 * The start position of the hole is defined by lblock, start_list, and
1780 * start_aligned. The end position of the hole is defined by lend,
1781 * end_list, and end_aligned.
1783 * start_aligned and end_aligned define down to which height the start
1784 * and end positions are aligned to the metadata tree (i.e., the
1785 * position is a multiple of the metadata granularity at the height
1786 * above). This determines at which heights additional meta pointers
1787 * needs to be preserved for the remaining data.
1791 u64 end_offset = offset + length;
1795 * Clip the end at the maximum file size for the given height:
1796 * that's how far the metadata goes; files bigger than that
1797 * will have additional layers of indirection.
1799 if (end_offset > maxsize)
1800 end_offset = maxsize;
1801 lend = end_offset >> bsize_shift;
1806 find_metapath(sdp, lend, &mp, ip->i_height);
1807 end_list = __end_list;
1808 memcpy(end_list, mp.mp_list, sizeof(mp.mp_list));
1810 for (mp_h = ip->i_height - 1; mp_h > 0; mp_h--) {
1817 find_metapath(sdp, lblock, &mp, ip->i_height);
1818 memcpy(start_list, mp.mp_list, sizeof(start_list));
1820 for (mp_h = ip->i_height - 1; mp_h > 0; mp_h--) {
1821 if (start_list[mp_h])
1824 start_aligned = mp_h;
1826 ret = gfs2_meta_inode_buffer(ip, &dibh);
1831 ret = lookup_metapath(ip, &mp);
1835 /* issue read-ahead on metadata */
1836 for (mp_h = 0; mp_h < mp.mp_aheight - 1; mp_h++) {
1837 metapointer_range(&mp, mp_h, start_list, start_aligned,
1838 end_list, end_aligned, &start, &end);
1839 gfs2_metapath_ra(ip->i_gl, start, end);
1842 if (mp.mp_aheight == ip->i_height)
1843 state = DEALLOC_MP_FULL; /* We have a complete metapath */
1845 state = DEALLOC_FILL_MP; /* deal with partial metapath */
1847 ret = gfs2_rindex_update(sdp);
1851 ret = gfs2_quota_hold(ip, NO_UID_QUOTA_CHANGE, NO_GID_QUOTA_CHANGE);
1854 gfs2_holder_mark_uninitialized(&rd_gh);
1858 while (state != DEALLOC_DONE) {
1860 /* Truncate a full metapath at the given strip height.
1861 * Note that strip_h == mp_h in order to be in this state. */
1862 case DEALLOC_MP_FULL:
1863 bh = mp.mp_bh[mp_h];
1864 gfs2_assert_withdraw(sdp, bh);
1865 if (gfs2_assert_withdraw(sdp,
1866 prev_bnr != bh->b_blocknr)) {
1867 printk(KERN_EMERG "GFS2: fsid=%s:inode %llu, "
1868 "block:%llu, i_h:%u, s_h:%u, mp_h:%u\n",
1870 (unsigned long long)ip->i_no_addr,
1871 prev_bnr, ip->i_height, strip_h, mp_h);
1873 prev_bnr = bh->b_blocknr;
1875 if (gfs2_metatype_check(sdp, bh,
1876 (mp_h ? GFS2_METATYPE_IN :
1877 GFS2_METATYPE_DI))) {
1883 * Below, passing end_aligned as 0 gives us the
1884 * metapointer range excluding the end point: the end
1885 * point is the first metapath we must not deallocate!
1888 metapointer_range(&mp, mp_h, start_list, start_aligned,
1889 end_list, 0 /* end_aligned */,
1891 ret = sweep_bh_for_rgrps(ip, &rd_gh, mp.mp_bh[mp_h],
1893 mp_h != ip->i_height - 1,
1896 /* If we hit an error or just swept dinode buffer,
1899 state = DEALLOC_DONE;
1902 state = DEALLOC_MP_LOWER;
1905 /* lower the metapath strip height */
1906 case DEALLOC_MP_LOWER:
1907 /* We're done with the current buffer, so release it,
1908 unless it's the dinode buffer. Then back up to the
1909 previous pointer. */
1911 brelse(mp.mp_bh[mp_h]);
1912 mp.mp_bh[mp_h] = NULL;
1914 /* If we can't get any lower in height, we've stripped
1915 off all we can. Next step is to back up and start
1916 stripping the previous level of metadata. */
1919 memcpy(mp.mp_list, start_list, sizeof(start_list));
1921 state = DEALLOC_FILL_MP;
1924 mp.mp_list[mp_h] = 0;
1925 mp_h--; /* search one metadata height down */
1927 if (walk_done(sdp, &mp, mp_h, end_list, end_aligned))
1929 /* Here we've found a part of the metapath that is not
1930 * allocated. We need to search at that height for the
1931 * next non-null pointer. */
1932 if (find_nonnull_ptr(sdp, &mp, mp_h, end_list, end_aligned)) {
1933 state = DEALLOC_FILL_MP;
1936 /* No more non-null pointers at this height. Back up
1937 to the previous height and try again. */
1938 break; /* loop around in the same state */
1940 /* Fill the metapath with buffers to the given height. */
1941 case DEALLOC_FILL_MP:
1942 /* Fill the buffers out to the current height. */
1943 ret = fillup_metapath(ip, &mp, mp_h);
1947 /* On the first pass, issue read-ahead on metadata. */
1948 if (mp.mp_aheight > 1 && strip_h == ip->i_height - 1) {
1949 unsigned int height = mp.mp_aheight - 1;
1951 /* No read-ahead for data blocks. */
1952 if (mp.mp_aheight - 1 == strip_h)
1955 for (; height >= mp.mp_aheight - ret; height--) {
1956 metapointer_range(&mp, height,
1957 start_list, start_aligned,
1958 end_list, end_aligned,
1960 gfs2_metapath_ra(ip->i_gl, start, end);
1964 /* If buffers found for the entire strip height */
1965 if (mp.mp_aheight - 1 == strip_h) {
1966 state = DEALLOC_MP_FULL;
1969 if (mp.mp_aheight < ip->i_height) /* We have a partial height */
1970 mp_h = mp.mp_aheight - 1;
1972 /* If we find a non-null block pointer, crawl a bit
1973 higher up in the metapath and try again, otherwise
1974 we need to look lower for a new starting point. */
1975 if (find_nonnull_ptr(sdp, &mp, mp_h, end_list, end_aligned))
1978 state = DEALLOC_MP_LOWER;
1984 if (current->journal_info == NULL) {
1985 ret = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS +
1989 down_write(&ip->i_rw_mutex);
1991 gfs2_statfs_change(sdp, 0, +btotal, 0);
1992 gfs2_quota_change(ip, -(s64)btotal, ip->i_inode.i_uid,
1994 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
1995 gfs2_trans_add_meta(ip->i_gl, dibh);
1996 gfs2_dinode_out(ip, dibh->b_data);
1997 up_write(&ip->i_rw_mutex);
1998 gfs2_trans_end(sdp);
2002 if (gfs2_holder_initialized(&rd_gh))
2003 gfs2_glock_dq_uninit(&rd_gh);
2004 if (current->journal_info) {
2005 up_write(&ip->i_rw_mutex);
2006 gfs2_trans_end(sdp);
2009 gfs2_quota_unhold(ip);
2011 release_metapath(&mp);
2015 static int trunc_end(struct gfs2_inode *ip)
2017 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2018 struct buffer_head *dibh;
2021 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
2025 down_write(&ip->i_rw_mutex);
2027 error = gfs2_meta_inode_buffer(ip, &dibh);
2031 if (!i_size_read(&ip->i_inode)) {
2033 ip->i_goal = ip->i_no_addr;
2034 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
2035 gfs2_ordered_del_inode(ip);
2037 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
2038 ip->i_diskflags &= ~GFS2_DIF_TRUNC_IN_PROG;
2040 gfs2_trans_add_meta(ip->i_gl, dibh);
2041 gfs2_dinode_out(ip, dibh->b_data);
2045 up_write(&ip->i_rw_mutex);
2046 gfs2_trans_end(sdp);
2051 * do_shrink - make a file smaller
2053 * @newsize: the size to make the file
2055 * Called with an exclusive lock on @inode. The @size must
2056 * be equal to or smaller than the current inode size.
2061 static int do_shrink(struct inode *inode, u64 newsize)
2063 struct gfs2_inode *ip = GFS2_I(inode);
2066 error = trunc_start(inode, newsize);
2069 if (gfs2_is_stuffed(ip))
2072 error = punch_hole(ip, newsize, 0);
2074 error = trunc_end(ip);
2079 void gfs2_trim_blocks(struct inode *inode)
2083 ret = do_shrink(inode, inode->i_size);
2088 * do_grow - Touch and update inode size
2090 * @size: The new size
2092 * This function updates the timestamps on the inode and
2093 * may also increase the size of the inode. This function
2094 * must not be called with @size any smaller than the current
2097 * Although it is not strictly required to unstuff files here,
2098 * earlier versions of GFS2 have a bug in the stuffed file reading
2099 * code which will result in a buffer overrun if the size is larger
2100 * than the max stuffed file size. In order to prevent this from
2101 * occurring, such files are unstuffed, but in other cases we can
2102 * just update the inode size directly.
2104 * Returns: 0 on success, or -ve on error
2107 static int do_grow(struct inode *inode, u64 size)
2109 struct gfs2_inode *ip = GFS2_I(inode);
2110 struct gfs2_sbd *sdp = GFS2_SB(inode);
2111 struct gfs2_alloc_parms ap = { .target = 1, };
2112 struct buffer_head *dibh;
2116 if (gfs2_is_stuffed(ip) && size > gfs2_max_stuffed_size(ip)) {
2117 error = gfs2_quota_lock_check(ip, &ap);
2121 error = gfs2_inplace_reserve(ip, &ap);
2123 goto do_grow_qunlock;
2127 error = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS + RES_RG_BIT +
2129 gfs2_is_jdata(ip) ? RES_JDATA : 0) +
2130 (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF ?
2133 goto do_grow_release;
2136 error = gfs2_unstuff_dinode(ip, NULL);
2141 error = gfs2_meta_inode_buffer(ip, &dibh);
2145 i_size_write(inode, size);
2146 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
2147 gfs2_trans_add_meta(ip->i_gl, dibh);
2148 gfs2_dinode_out(ip, dibh->b_data);
2152 gfs2_trans_end(sdp);
2155 gfs2_inplace_release(ip);
2157 gfs2_quota_unlock(ip);
2163 * gfs2_setattr_size - make a file a given size
2165 * @newsize: the size to make the file
2167 * The file size can grow, shrink, or stay the same size. This
2168 * is called holding i_rwsem and an exclusive glock on the inode
2174 int gfs2_setattr_size(struct inode *inode, u64 newsize)
2176 struct gfs2_inode *ip = GFS2_I(inode);
2179 BUG_ON(!S_ISREG(inode->i_mode));
2181 ret = inode_newsize_ok(inode, newsize);
2185 inode_dio_wait(inode);
2187 ret = gfs2_rsqa_alloc(ip);
2191 if (newsize >= inode->i_size) {
2192 ret = do_grow(inode, newsize);
2196 ret = do_shrink(inode, newsize);
2198 gfs2_rsqa_delete(ip, NULL);
2202 int gfs2_truncatei_resume(struct gfs2_inode *ip)
2205 error = punch_hole(ip, i_size_read(&ip->i_inode), 0);
2207 error = trunc_end(ip);
2211 int gfs2_file_dealloc(struct gfs2_inode *ip)
2213 return punch_hole(ip, 0, 0);
2217 * gfs2_free_journal_extents - Free cached journal bmap info
2222 void gfs2_free_journal_extents(struct gfs2_jdesc *jd)
2224 struct gfs2_journal_extent *jext;
2226 while(!list_empty(&jd->extent_list)) {
2227 jext = list_entry(jd->extent_list.next, struct gfs2_journal_extent, list);
2228 list_del(&jext->list);
2234 * gfs2_add_jextent - Add or merge a new extent to extent cache
2235 * @jd: The journal descriptor
2236 * @lblock: The logical block at start of new extent
2237 * @dblock: The physical block at start of new extent
2238 * @blocks: Size of extent in fs blocks
2240 * Returns: 0 on success or -ENOMEM
2243 static int gfs2_add_jextent(struct gfs2_jdesc *jd, u64 lblock, u64 dblock, u64 blocks)
2245 struct gfs2_journal_extent *jext;
2247 if (!list_empty(&jd->extent_list)) {
2248 jext = list_entry(jd->extent_list.prev, struct gfs2_journal_extent, list);
2249 if ((jext->dblock + jext->blocks) == dblock) {
2250 jext->blocks += blocks;
2255 jext = kzalloc(sizeof(struct gfs2_journal_extent), GFP_NOFS);
2258 jext->dblock = dblock;
2259 jext->lblock = lblock;
2260 jext->blocks = blocks;
2261 list_add_tail(&jext->list, &jd->extent_list);
2267 * gfs2_map_journal_extents - Cache journal bmap info
2268 * @sdp: The super block
2269 * @jd: The journal to map
2271 * Create a reusable "extent" mapping from all logical
2272 * blocks to all physical blocks for the given journal. This will save
2273 * us time when writing journal blocks. Most journals will have only one
2274 * extent that maps all their logical blocks. That's because gfs2.mkfs
2275 * arranges the journal blocks sequentially to maximize performance.
2276 * So the extent would map the first block for the entire file length.
2277 * However, gfs2_jadd can happen while file activity is happening, so
2278 * those journals may not be sequential. Less likely is the case where
2279 * the users created their own journals by mounting the metafs and
2280 * laying it out. But it's still possible. These journals might have
2283 * Returns: 0 on success, or error on failure
2286 int gfs2_map_journal_extents(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd)
2290 struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
2291 struct buffer_head bh;
2292 unsigned int shift = sdp->sd_sb.sb_bsize_shift;
2297 start = ktime_get();
2298 lblock_stop = i_size_read(jd->jd_inode) >> shift;
2299 size = (lblock_stop - lblock) << shift;
2301 WARN_ON(!list_empty(&jd->extent_list));
2307 rc = gfs2_block_map(jd->jd_inode, lblock, &bh, 0);
2308 if (rc || !buffer_mapped(&bh))
2310 rc = gfs2_add_jextent(jd, lblock, bh.b_blocknr, bh.b_size >> shift);
2314 lblock += (bh.b_size >> ip->i_inode.i_blkbits);
2318 fs_info(sdp, "journal %d mapped with %u extents in %lldms\n", jd->jd_jid,
2319 jd->nr_extents, ktime_ms_delta(end, start));
2323 fs_warn(sdp, "error %d mapping journal %u at offset %llu (extent %u)\n",
2325 (unsigned long long)(i_size_read(jd->jd_inode) - size),
2327 fs_warn(sdp, "bmap=%d lblock=%llu block=%llu, state=0x%08lx, size=%llu\n",
2328 rc, (unsigned long long)lblock, (unsigned long long)bh.b_blocknr,
2329 bh.b_state, (unsigned long long)bh.b_size);
2330 gfs2_free_journal_extents(jd);
2335 * gfs2_write_alloc_required - figure out if a write will require an allocation
2336 * @ip: the file being written to
2337 * @offset: the offset to write to
2338 * @len: the number of bytes being written
2340 * Returns: 1 if an alloc is required, 0 otherwise
2343 int gfs2_write_alloc_required(struct gfs2_inode *ip, u64 offset,
2346 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2347 struct buffer_head bh;
2349 u64 lblock, lblock_stop, size;
2355 if (gfs2_is_stuffed(ip)) {
2356 if (offset + len > gfs2_max_stuffed_size(ip))
2361 shift = sdp->sd_sb.sb_bsize_shift;
2362 BUG_ON(gfs2_is_dir(ip));
2363 end_of_file = (i_size_read(&ip->i_inode) + sdp->sd_sb.sb_bsize - 1) >> shift;
2364 lblock = offset >> shift;
2365 lblock_stop = (offset + len + sdp->sd_sb.sb_bsize - 1) >> shift;
2366 if (lblock_stop > end_of_file && ip != GFS2_I(sdp->sd_rindex))
2369 size = (lblock_stop - lblock) << shift;
2373 gfs2_block_map(&ip->i_inode, lblock, &bh, 0);
2374 if (!buffer_mapped(&bh))
2377 lblock += (bh.b_size >> ip->i_inode.i_blkbits);
2383 static int stuffed_zero_range(struct inode *inode, loff_t offset, loff_t length)
2385 struct gfs2_inode *ip = GFS2_I(inode);
2386 struct buffer_head *dibh;
2389 if (offset >= inode->i_size)
2391 if (offset + length > inode->i_size)
2392 length = inode->i_size - offset;
2394 error = gfs2_meta_inode_buffer(ip, &dibh);
2397 gfs2_trans_add_meta(ip->i_gl, dibh);
2398 memset(dibh->b_data + sizeof(struct gfs2_dinode) + offset, 0,
2404 static int gfs2_journaled_truncate_range(struct inode *inode, loff_t offset,
2407 struct gfs2_sbd *sdp = GFS2_SB(inode);
2408 loff_t max_chunk = GFS2_JTRUNC_REVOKES * sdp->sd_vfs->s_blocksize;
2412 struct gfs2_trans *tr;
2417 if (chunk > max_chunk)
2420 offs = offset & ~PAGE_MASK;
2421 if (offs && chunk > PAGE_SIZE)
2422 chunk = offs + ((chunk - offs) & PAGE_MASK);
2424 truncate_pagecache_range(inode, offset, chunk);
2428 tr = current->journal_info;
2429 if (!test_bit(TR_TOUCHED, &tr->tr_flags))
2432 gfs2_trans_end(sdp);
2433 error = gfs2_trans_begin(sdp, RES_DINODE, GFS2_JTRUNC_REVOKES);
2440 int __gfs2_punch_hole(struct file *file, loff_t offset, loff_t length)
2442 struct inode *inode = file_inode(file);
2443 struct gfs2_inode *ip = GFS2_I(inode);
2444 struct gfs2_sbd *sdp = GFS2_SB(inode);
2447 if (gfs2_is_jdata(ip))
2448 error = gfs2_trans_begin(sdp, RES_DINODE + 2 * RES_JDATA,
2449 GFS2_JTRUNC_REVOKES);
2451 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
2455 if (gfs2_is_stuffed(ip)) {
2456 error = stuffed_zero_range(inode, offset, length);
2460 unsigned int start_off, end_len, blocksize;
2462 blocksize = i_blocksize(inode);
2463 start_off = offset & (blocksize - 1);
2464 end_len = (offset + length) & (blocksize - 1);
2466 unsigned int len = length;
2467 if (length > blocksize - start_off)
2468 len = blocksize - start_off;
2469 error = gfs2_block_zero_range(inode, offset, len);
2472 if (start_off + length < blocksize)
2476 error = gfs2_block_zero_range(inode,
2477 offset + length - end_len, end_len);
2483 if (gfs2_is_jdata(ip)) {
2484 BUG_ON(!current->journal_info);
2485 gfs2_journaled_truncate_range(inode, offset, length);
2487 truncate_pagecache_range(inode, offset, offset + length - 1);
2489 file_update_time(file);
2490 mark_inode_dirty(inode);
2492 if (current->journal_info)
2493 gfs2_trans_end(sdp);
2495 if (!gfs2_is_stuffed(ip))
2496 error = punch_hole(ip, offset, length);
2499 if (current->journal_info)
2500 gfs2_trans_end(sdp);