fs/xfs/libxfs/xfs_bmap.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2000-2006 Silicon Graphics, Inc.
   4  * All Rights Reserved.
   5  */
   6 #include "xfs.h"
   7 #include "xfs_fs.h"
   8 #include "xfs_shared.h"
   9 #include "xfs_format.h"
  10 #include "xfs_log_format.h"
  11 #include "xfs_trans_resv.h"
  12 #include "xfs_bit.h"
  13 #include "xfs_sb.h"
  14 #include "xfs_mount.h"
  15 #include "xfs_defer.h"
  16 #include "xfs_dir2.h"
  17 #include "xfs_inode.h"
  18 #include "xfs_btree.h"
  19 #include "xfs_trans.h"
  20 #include "xfs_alloc.h"
  21 #include "xfs_bmap.h"
  22 #include "xfs_bmap_util.h"
  23 #include "xfs_bmap_btree.h"
  24 #include "xfs_rtbitmap.h"
  25 #include "xfs_errortag.h"
  26 #include "xfs_error.h"
  27 #include "xfs_quota.h"
  28 #include "xfs_trans_space.h"
  29 #include "xfs_buf_item.h"
  30 #include "xfs_trace.h"
  31 #include "xfs_attr_leaf.h"
  32 #include "xfs_filestream.h"
  33 #include "xfs_rmap.h"
  34 #include "xfs_ag.h"
  35 #include "xfs_ag_resv.h"
  36 #include "xfs_refcount.h"
  37 #include "xfs_icache.h"
  38 #include "xfs_iomap.h"
  39 #include "xfs_health.h"
  40 #include "xfs_bmap_item.h"
  41 #include "xfs_symlink_remote.h"
  42
  43 struct kmem_cache               *xfs_bmap_intent_cache;
  44
  45 /*
  46  * Miscellaneous helper functions
  47  */
  48
  49 /*
  50  * Compute and fill in the value of the maximum depth of a bmap btree
  51  * in this filesystem.  Done once, during mount.
  52  */
  53 void
  54 xfs_bmap_compute_maxlevels(
  55         xfs_mount_t     *mp,            /* file system mount structure */
  56         int             whichfork)      /* data or attr fork */
  57 {
  58         uint64_t        maxblocks;      /* max blocks at this level */
  59         xfs_extnum_t    maxleafents;    /* max leaf entries possible */
  60         int             level;          /* btree level */
  61         int             maxrootrecs;    /* max records in root block */
  62         int             minleafrecs;    /* min records in leaf block */
  63         int             minnoderecs;    /* min records in node block */
  64         int             sz;             /* root block size */
  65
  66         /*
  67          * The maximum number of extents in a fork, hence the maximum number of
  68          * leaf entries, is controlled by the size of the on-disk extent count.
  69          *
  70          * Note that we can no longer assume that if we are in ATTR1 that the
  71          * fork offset of all the inodes will be
  72          * (xfs_default_attroffset(ip) >> 3) because we could have mounted with
  73          * ATTR2 and then mounted back with ATTR1, keeping the i_forkoff's fixed
  74          * but probably at various positions. Therefore, for both ATTR1 and
  75          * ATTR2 we have to assume the worst case scenario of a minimum size
  76          * available.
  77          */
  78         maxleafents = xfs_iext_max_nextents(xfs_has_large_extent_counts(mp),
  79                                 whichfork);
  80         if (whichfork == XFS_DATA_FORK)
  81                 sz = XFS_BMDR_SPACE_CALC(MINDBTPTRS);
  82         else
  83                 sz = XFS_BMDR_SPACE_CALC(MINABTPTRS);
  84
  85         maxrootrecs = xfs_bmdr_maxrecs(sz, 0);
  86         minleafrecs = mp->m_bmap_dmnr[0];
  87         minnoderecs = mp->m_bmap_dmnr[1];
  88         maxblocks = howmany_64(maxleafents, minleafrecs);
  89         for (level = 1; maxblocks > 1; level++) {
  90                 if (maxblocks <= maxrootrecs)
  91                         maxblocks = 1;
  92                 else
  93                         maxblocks = howmany_64(maxblocks, minnoderecs);
  94         }
  95         mp->m_bm_maxlevels[whichfork] = level;
  96         ASSERT(mp->m_bm_maxlevels[whichfork] <= xfs_bmbt_maxlevels_ondisk());
  97 }
  98
  99 unsigned int
 100 xfs_bmap_compute_attr_offset(
 101         struct xfs_mount        *mp)
 102 {
 103         if (mp->m_sb.sb_inodesize == 256)
 104                 return XFS_LITINO(mp) - XFS_BMDR_SPACE_CALC(MINABTPTRS);
 105         return XFS_BMDR_SPACE_CALC(6 * MINABTPTRS);
 106 }
 107
 108 STATIC int                              /* error */
 109 xfs_bmbt_lookup_eq(
 110         struct xfs_btree_cur    *cur,
 111         struct xfs_bmbt_irec    *irec,
 112         int                     *stat)  /* success/failure */
 113 {
 114         cur->bc_rec.b = *irec;
 115         return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat);
 116 }
 117
 118 STATIC int                              /* error */
 119 xfs_bmbt_lookup_first(
 120         struct xfs_btree_cur    *cur,
 121         int                     *stat)  /* success/failure */
 122 {
 123         cur->bc_rec.b.br_startoff = 0;
 124         cur->bc_rec.b.br_startblock = 0;
 125         cur->bc_rec.b.br_blockcount = 0;
 126         return xfs_btree_lookup(cur, XFS_LOOKUP_GE, stat);
 127 }
 128
 129 /*
 130  * Check if the inode needs to be converted to btree format.
 131  */
 132 static inline bool xfs_bmap_needs_btree(struct xfs_inode *ip, int whichfork)
 133 {
 134         struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork);
 135
 136         return whichfork != XFS_COW_FORK &&
 137                 ifp->if_format == XFS_DINODE_FMT_EXTENTS &&
 138                 ifp->if_nextents > XFS_IFORK_MAXEXT(ip, whichfork);
 139 }
 140
 141 /*
 142  * Check if the inode should be converted to extent format.
 143  */
 144 static inline bool xfs_bmap_wants_extents(struct xfs_inode *ip, int whichfork)
 145 {
 146         struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork);
 147
 148         return whichfork != XFS_COW_FORK &&
 149                 ifp->if_format == XFS_DINODE_FMT_BTREE &&
 150                 ifp->if_nextents <= XFS_IFORK_MAXEXT(ip, whichfork);
 151 }
 152
 153 /*
 154  * Update the record referred to by cur to the value given by irec
 155  * This either works (return 0) or gets an EFSCORRUPTED error.
 156  */
 157 STATIC int
 158 xfs_bmbt_update(
 159         struct xfs_btree_cur    *cur,
 160         struct xfs_bmbt_irec    *irec)
 161 {
 162         union xfs_btree_rec     rec;
 163
 164         xfs_bmbt_disk_set_all(&rec.bmbt, irec);
 165         return xfs_btree_update(cur, &rec);
 166 }
 167
 168 /*
 169  * Compute the worst-case number of indirect blocks that will be used
 170  * for ip's delayed extent of length "len".
 171  */
 172 STATIC xfs_filblks_t
 173 xfs_bmap_worst_indlen(
 174         xfs_inode_t     *ip,            /* incore inode pointer */
 175         xfs_filblks_t   len)            /* delayed extent length */
 176 {
 177         int             level;          /* btree level number */
 178         int             maxrecs;        /* maximum record count at this level */
 179         xfs_mount_t     *mp;            /* mount structure */
 180         xfs_filblks_t   rval;           /* return value */
 181
 182         mp = ip->i_mount;
 183         maxrecs = mp->m_bmap_dmxr[0];
 184         for (level = 0, rval = 0;
 185              level < XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK);
 186              level++) {
 187                 len += maxrecs - 1;
 188                 do_div(len, maxrecs);
 189                 rval += len;
 190                 if (len == 1)
 191                         return rval + XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK) -
 192                                 level - 1;
 193                 if (level == 0)
 194                         maxrecs = mp->m_bmap_dmxr[1];
 195         }
 196         return rval;
 197 }
 198
 199 /*
 200  * Calculate the default attribute fork offset for newly created inodes.
 201  */
 202 uint
 203 xfs_default_attroffset(
 204         struct xfs_inode        *ip)
 205 {
 206         if (ip->i_df.if_format == XFS_DINODE_FMT_DEV)
 207                 return roundup(sizeof(xfs_dev_t), 8);
 208         return M_IGEO(ip->i_mount)->attr_fork_offset;
 209 }
 210
 211 /*
 212  * Helper routine to reset inode i_forkoff field when switching attribute fork
 213  * from local to extent format - we reset it where possible to make space
 214  * available for inline data fork extents.
 215  */
 216 STATIC void
 217 xfs_bmap_forkoff_reset(
 218         xfs_inode_t     *ip,
 219         int             whichfork)
 220 {
 221         if (whichfork == XFS_ATTR_FORK &&
 222             ip->i_df.if_format != XFS_DINODE_FMT_DEV &&
 223             ip->i_df.if_format != XFS_DINODE_FMT_BTREE) {
 224                 uint    dfl_forkoff = xfs_default_attroffset(ip) >> 3;
 225
 226                 if (dfl_forkoff > ip->i_forkoff)
 227                         ip->i_forkoff = dfl_forkoff;
 228         }
 229 }
 230
 231 static int
 232 xfs_bmap_read_buf(
 233         struct xfs_mount        *mp,            /* file system mount point */
 234         struct xfs_trans        *tp,            /* transaction pointer */
 235         xfs_fsblock_t           fsbno,          /* file system block number */
 236         struct xfs_buf          **bpp)          /* buffer for fsbno */
 237 {
 238         struct xfs_buf          *bp;            /* return value */
 239         int                     error;
 240
 241         if (!xfs_verify_fsbno(mp, fsbno))
 242                 return -EFSCORRUPTED;
 243         error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp,
 244                         XFS_FSB_TO_DADDR(mp, fsbno), mp->m_bsize, 0, &bp,
 245                         &xfs_bmbt_buf_ops);
 246         if (!error) {
 247                 xfs_buf_set_ref(bp, XFS_BMAP_BTREE_REF);
 248                 *bpp = bp;
 249         }
 250         return error;
 251 }
 252
 253 #ifdef DEBUG
 254 STATIC struct xfs_buf *
 255 xfs_bmap_get_bp(
 256         struct xfs_btree_cur    *cur,
 257         xfs_fsblock_t           bno)
 258 {
 259         struct xfs_log_item     *lip;
 260         int                     i;
 261
 262         if (!cur)
 263                 return NULL;
 264
 265         for (i = 0; i < cur->bc_maxlevels; i++) {
 266                 if (!cur->bc_levels[i].bp)
 267                         break;
 268                 if (xfs_buf_daddr(cur->bc_levels[i].bp) == bno)
 269                         return cur->bc_levels[i].bp;
 270         }
 271
 272         /* Chase down all the log items to see if the bp is there */
 273         list_for_each_entry(lip, &cur->bc_tp->t_items, li_trans) {
 274                 struct xfs_buf_log_item *bip = (struct xfs_buf_log_item *)lip;
 275
 276                 if (bip->bli_item.li_type == XFS_LI_BUF &&
 277                     xfs_buf_daddr(bip->bli_buf) == bno)
 278                         return bip->bli_buf;
 279         }
 280
 281         return NULL;
 282 }
 283
 284 STATIC void
 285 xfs_check_block(
 286         struct xfs_btree_block  *block,
 287         xfs_mount_t             *mp,
 288         int                     root,
 289         short                   sz)
 290 {
 291         int                     i, j, dmxr;
 292         __be64                  *pp, *thispa;   /* pointer to block address */
 293         xfs_bmbt_key_t          *prevp, *keyp;
 294
 295         ASSERT(be16_to_cpu(block->bb_level) > 0);
 296
 297         prevp = NULL;
 298         for( i = 1; i <= xfs_btree_get_numrecs(block); i++) {
 299                 dmxr = mp->m_bmap_dmxr[0];
 300                 keyp = XFS_BMBT_KEY_ADDR(mp, block, i);
 301
 302                 if (prevp) {
 303                         ASSERT(be64_to_cpu(prevp->br_startoff) <
 304                                be64_to_cpu(keyp->br_startoff));
 305                 }
 306                 prevp = keyp;
 307
 308                 /*
 309                  * Compare the block numbers to see if there are dups.
 310                  */
 311                 if (root)
 312                         pp = XFS_BMAP_BROOT_PTR_ADDR(mp, block, i, sz);
 313                 else
 314                         pp = XFS_BMBT_PTR_ADDR(mp, block, i, dmxr);
 315
 316                 for (j = i+1; j <= be16_to_cpu(block->bb_numrecs); j++) {
 317                         if (root)
 318                                 thispa = XFS_BMAP_BROOT_PTR_ADDR(mp, block, j, sz);
 319                         else
 320                                 thispa = XFS_BMBT_PTR_ADDR(mp, block, j, dmxr);
 321                         if (*thispa == *pp) {
 322                                 xfs_warn(mp, "%s: thispa(%d) == pp(%d) %lld",
 323                                         __func__, j, i,
 324                                         (unsigned long long)be64_to_cpu(*thispa));
 325                                 xfs_err(mp, "%s: ptrs are equal in node\n",
 326                                         __func__);
 327                                 xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE);
 328                         }
 329                 }
 330         }
 331 }
 332
 333 /*
 334  * Check that the extents for the inode ip are in the right order in all
 335  * btree leaves. THis becomes prohibitively expensive for large extent count
 336  * files, so don't bother with inodes that have more than 10,000 extents in
 337  * them. The btree record ordering checks will still be done, so for such large
 338  * bmapbt constructs that is going to catch most corruptions.
 339  */
 340 STATIC void
 341 xfs_bmap_check_leaf_extents(
 342         struct xfs_btree_cur    *cur,   /* btree cursor or null */
 343         xfs_inode_t             *ip,            /* incore inode pointer */
 344         int                     whichfork)      /* data or attr fork */
 345 {
 346         struct xfs_mount        *mp = ip->i_mount;
 347         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
 348         struct xfs_btree_block  *block; /* current btree block */
 349         xfs_fsblock_t           bno;    /* block # of "block" */
 350         struct xfs_buf          *bp;    /* buffer for "block" */
 351         int                     error;  /* error return value */
 352         xfs_extnum_t            i=0, j; /* index into the extents list */
 353         int                     level;  /* btree level, for checking */
 354         __be64                  *pp;    /* pointer to block address */
 355         xfs_bmbt_rec_t          *ep;    /* pointer to current extent */
 356         xfs_bmbt_rec_t          last = {0, 0}; /* last extent in prev block */
 357         xfs_bmbt_rec_t          *nextp; /* pointer to next extent */
 358         int                     bp_release = 0;
 359
 360         if (ifp->if_format != XFS_DINODE_FMT_BTREE)
 361                 return;
 362
 363         /* skip large extent count inodes */
 364         if (ip->i_df.if_nextents > 10000)
 365                 return;
 366
 367         bno = NULLFSBLOCK;
 368         block = ifp->if_broot;
 369         /*
 370          * Root level must use BMAP_BROOT_PTR_ADDR macro to get ptr out.
 371          */
 372         level = be16_to_cpu(block->bb_level);
 373         ASSERT(level > 0);
 374         xfs_check_block(block, mp, 1, ifp->if_broot_bytes);
 375         pp = XFS_BMAP_BROOT_PTR_ADDR(mp, block, 1, ifp->if_broot_bytes);
 376         bno = be64_to_cpu(*pp);
 377
 378         ASSERT(bno != NULLFSBLOCK);
 379         ASSERT(XFS_FSB_TO_AGNO(mp, bno) < mp->m_sb.sb_agcount);
 380         ASSERT(XFS_FSB_TO_AGBNO(mp, bno) < mp->m_sb.sb_agblocks);
 381
 382         /*
 383          * Go down the tree until leaf level is reached, following the first
 384          * pointer (leftmost) at each level.
 385          */
 386         while (level-- > 0) {
 387                 /* See if buf is in cur first */
 388                 bp_release = 0;
 389                 bp = xfs_bmap_get_bp(cur, XFS_FSB_TO_DADDR(mp, bno));
 390                 if (!bp) {
 391                         bp_release = 1;
 392                         error = xfs_bmap_read_buf(mp, NULL, bno, &bp);
 393                         if (xfs_metadata_is_sick(error))
 394                                 xfs_btree_mark_sick(cur);
 395                         if (error)
 396                                 goto error_norelse;
 397                 }
 398                 block = XFS_BUF_TO_BLOCK(bp);
 399                 if (level == 0)
 400                         break;
 401
 402                 /*
 403                  * Check this block for basic sanity (increasing keys and
 404                  * no duplicate blocks).
 405                  */
 406
 407                 xfs_check_block(block, mp, 0, 0);
 408                 pp = XFS_BMBT_PTR_ADDR(mp, block, 1, mp->m_bmap_dmxr[1]);
 409                 bno = be64_to_cpu(*pp);
 410                 if (XFS_IS_CORRUPT(mp, !xfs_verify_fsbno(mp, bno))) {
 411                         xfs_btree_mark_sick(cur);
 412                         error = -EFSCORRUPTED;
 413                         goto error0;
 414                 }
 415                 if (bp_release) {
 416                         bp_release = 0;
 417                         xfs_trans_brelse(NULL, bp);
 418                 }
 419         }
 420
 421         /*
 422          * Here with bp and block set to the leftmost leaf node in the tree.
 423          */
 424         i = 0;
 425
 426         /*
 427          * Loop over all leaf nodes checking that all extents are in the right order.
 428          */
 429         for (;;) {
 430                 xfs_fsblock_t   nextbno;
 431                 xfs_extnum_t    num_recs;
 432
 433
 434                 num_recs = xfs_btree_get_numrecs(block);
 435
 436                 /*
 437                  * Read-ahead the next leaf block, if any.
 438                  */
 439
 440                 nextbno = be64_to_cpu(block->bb_u.l.bb_rightsib);
 441
 442                 /*
 443                  * Check all the extents to make sure they are OK.
 444                  * If we had a previous block, the last entry should
 445                  * conform with the first entry in this one.
 446                  */
 447
 448                 ep = XFS_BMBT_REC_ADDR(mp, block, 1);
 449                 if (i) {
 450                         ASSERT(xfs_bmbt_disk_get_startoff(&last) +
 451                                xfs_bmbt_disk_get_blockcount(&last) <=
 452                                xfs_bmbt_disk_get_startoff(ep));
 453                 }
 454                 for (j = 1; j < num_recs; j++) {
 455                         nextp = XFS_BMBT_REC_ADDR(mp, block, j + 1);
 456                         ASSERT(xfs_bmbt_disk_get_startoff(ep) +
 457                                xfs_bmbt_disk_get_blockcount(ep) <=
 458                                xfs_bmbt_disk_get_startoff(nextp));
 459                         ep = nextp;
 460                 }
 461
 462                 last = *ep;
 463                 i += num_recs;
 464                 if (bp_release) {
 465                         bp_release = 0;
 466                         xfs_trans_brelse(NULL, bp);
 467                 }
 468                 bno = nextbno;
 469                 /*
 470                  * If we've reached the end, stop.
 471                  */
 472                 if (bno == NULLFSBLOCK)
 473                         break;
 474
 475                 bp_release = 0;
 476                 bp = xfs_bmap_get_bp(cur, XFS_FSB_TO_DADDR(mp, bno));
 477                 if (!bp) {
 478                         bp_release = 1;
 479                         error = xfs_bmap_read_buf(mp, NULL, bno, &bp);
 480                         if (xfs_metadata_is_sick(error))
 481                                 xfs_btree_mark_sick(cur);
 482                         if (error)
 483                                 goto error_norelse;
 484                 }
 485                 block = XFS_BUF_TO_BLOCK(bp);
 486         }
 487
 488         return;
 489
 490 error0:
 491         xfs_warn(mp, "%s: at error0", __func__);
 492         if (bp_release)
 493                 xfs_trans_brelse(NULL, bp);
 494 error_norelse:
 495         xfs_warn(mp, "%s: BAD after btree leaves for %llu extents",
 496                 __func__, i);
 497         xfs_err(mp, "%s: CORRUPTED BTREE OR SOMETHING", __func__);
 498         xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE);
 499         return;
 500 }
 501
 502 /*
 503  * Validate that the bmbt_irecs being returned from bmapi are valid
 504  * given the caller's original parameters.  Specifically check the
 505  * ranges of the returned irecs to ensure that they only extend beyond
 506  * the given parameters if the XFS_BMAPI_ENTIRE flag was set.
 507  */
 508 STATIC void
 509 xfs_bmap_validate_ret(
 510         xfs_fileoff_t           bno,
 511         xfs_filblks_t           len,
 512         uint32_t                flags,
 513         xfs_bmbt_irec_t         *mval,
 514         int                     nmap,
 515         int                     ret_nmap)
 516 {
 517         int                     i;              /* index to map values */
 518
 519         ASSERT(ret_nmap <= nmap);
 520
 521         for (i = 0; i < ret_nmap; i++) {
 522                 ASSERT(mval[i].br_blockcount > 0);
 523                 if (!(flags & XFS_BMAPI_ENTIRE)) {
 524                         ASSERT(mval[i].br_startoff >= bno);
 525                         ASSERT(mval[i].br_blockcount <= len);
 526                         ASSERT(mval[i].br_startoff + mval[i].br_blockcount <=
 527                                bno + len);
 528                 } else {
 529                         ASSERT(mval[i].br_startoff < bno + len);
 530                         ASSERT(mval[i].br_startoff + mval[i].br_blockcount >
 531                                bno);
 532                 }
 533                 ASSERT(i == 0 ||
 534                        mval[i - 1].br_startoff + mval[i - 1].br_blockcount ==
 535                        mval[i].br_startoff);
 536                 ASSERT(mval[i].br_startblock != DELAYSTARTBLOCK &&
 537                        mval[i].br_startblock != HOLESTARTBLOCK);
 538                 ASSERT(mval[i].br_state == XFS_EXT_NORM ||
 539                        mval[i].br_state == XFS_EXT_UNWRITTEN);
 540         }
 541 }
 542
 543 #else
 544 #define xfs_bmap_check_leaf_extents(cur, ip, whichfork)         do { } while (0)
 545 #define xfs_bmap_validate_ret(bno,len,flags,mval,onmap,nmap)    do { } while (0)
 546 #endif /* DEBUG */
 547
 548 /*
 549  * Inode fork format manipulation functions
 550  */
 551
 552 /*
 553  * Convert the inode format to extent format if it currently is in btree format,
 554  * but the extent list is small enough that it fits into the extent format.
 555  *
 556  * Since the extents are already in-core, all we have to do is give up the space
 557  * for the btree root and pitch the leaf block.
 558  */
 559 STATIC int                              /* error */
 560 xfs_bmap_btree_to_extents(
 561         struct xfs_trans        *tp,    /* transaction pointer */
 562         struct xfs_inode        *ip,    /* incore inode pointer */
 563         struct xfs_btree_cur    *cur,   /* btree cursor */
 564         int                     *logflagsp, /* inode logging flags */
 565         int                     whichfork)  /* data or attr fork */
 566 {
 567         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
 568         struct xfs_mount        *mp = ip->i_mount;
 569         struct xfs_btree_block  *rblock = ifp->if_broot;
 570         struct xfs_btree_block  *cblock;/* child btree block */
 571         xfs_fsblock_t           cbno;   /* child block number */
 572         struct xfs_buf          *cbp;   /* child block's buffer */
 573         int                     error;  /* error return value */
 574         __be64                  *pp;    /* ptr to block address */
 575         struct xfs_owner_info   oinfo;
 576
 577         /* check if we actually need the extent format first: */
 578         if (!xfs_bmap_wants_extents(ip, whichfork))
 579                 return 0;
 580
 581         ASSERT(cur);
 582         ASSERT(whichfork != XFS_COW_FORK);
 583         ASSERT(ifp->if_format == XFS_DINODE_FMT_BTREE);
 584         ASSERT(be16_to_cpu(rblock->bb_level) == 1);
 585         ASSERT(be16_to_cpu(rblock->bb_numrecs) == 1);
 586         ASSERT(xfs_bmbt_maxrecs(mp, ifp->if_broot_bytes, 0) == 1);
 587
 588         pp = XFS_BMAP_BROOT_PTR_ADDR(mp, rblock, 1, ifp->if_broot_bytes);
 589         cbno = be64_to_cpu(*pp);
 590 #ifdef DEBUG
 591         if (XFS_IS_CORRUPT(cur->bc_mp, !xfs_verify_fsbno(mp, cbno))) {
 592                 xfs_btree_mark_sick(cur);
 593                 return -EFSCORRUPTED;
 594         }
 595 #endif
 596         error = xfs_bmap_read_buf(mp, tp, cbno, &cbp);
 597         if (xfs_metadata_is_sick(error))
 598                 xfs_btree_mark_sick(cur);
 599         if (error)
 600                 return error;
 601         cblock = XFS_BUF_TO_BLOCK(cbp);
 602         if ((error = xfs_btree_check_block(cur, cblock, 0, cbp)))
 603                 return error;
 604
 605         xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, whichfork);
 606         error = xfs_free_extent_later(cur->bc_tp, cbno, 1, &oinfo,
 607                         XFS_AG_RESV_NONE, false);
 608         if (error)
 609                 return error;
 610
 611         ip->i_nblocks--;
 612         xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, -1L);
 613         xfs_trans_binval(tp, cbp);
 614         if (cur->bc_levels[0].bp == cbp)
 615                 cur->bc_levels[0].bp = NULL;
 616         xfs_iroot_realloc(ip, -1, whichfork);
 617         ASSERT(ifp->if_broot == NULL);
 618         ifp->if_format = XFS_DINODE_FMT_EXTENTS;
 619         *logflagsp |= XFS_ILOG_CORE | xfs_ilog_fext(whichfork);
 620         return 0;
 621 }
 622
 623 /*
 624  * Convert an extents-format file into a btree-format file.
 625  * The new file will have a root block (in the inode) and a single child block.
 626  */
 627 STATIC int                                      /* error */
 628 xfs_bmap_extents_to_btree(
 629         struct xfs_trans        *tp,            /* transaction pointer */
 630         struct xfs_inode        *ip,            /* incore inode pointer */
 631         struct xfs_btree_cur    **curp,         /* cursor returned to caller */
 632         int                     wasdel,         /* converting a delayed alloc */
 633         int                     *logflagsp,     /* inode logging flags */
 634         int                     whichfork)      /* data or attr fork */
 635 {
 636         struct xfs_btree_block  *ablock;        /* allocated (child) bt block */
 637         struct xfs_buf          *abp;           /* buffer for ablock */
 638         struct xfs_alloc_arg    args;           /* allocation arguments */
 639         struct xfs_bmbt_rec     *arp;           /* child record pointer */
 640         struct xfs_btree_block  *block;         /* btree root block */
 641         struct xfs_btree_cur    *cur;           /* bmap btree cursor */
 642         int                     error;          /* error return value */
 643         struct xfs_ifork        *ifp;           /* inode fork pointer */
 644         struct xfs_bmbt_key     *kp;            /* root block key pointer */
 645         struct xfs_mount        *mp;            /* mount structure */
 646         xfs_bmbt_ptr_t          *pp;            /* root block address pointer */
 647         struct xfs_iext_cursor  icur;
 648         struct xfs_bmbt_irec    rec;
 649         xfs_extnum_t            cnt = 0;
 650
 651         mp = ip->i_mount;
 652         ASSERT(whichfork != XFS_COW_FORK);
 653         ifp = xfs_ifork_ptr(ip, whichfork);
 654         ASSERT(ifp->if_format == XFS_DINODE_FMT_EXTENTS);
 655
 656         /*
 657          * Make space in the inode incore. This needs to be undone if we fail
 658          * to expand the root.
 659          */
 660         xfs_iroot_realloc(ip, 1, whichfork);
 661
 662         /*
 663          * Fill in the root.
 664          */
 665         block = ifp->if_broot;
 666         xfs_bmbt_init_block(ip, block, NULL, 1, 1);
 667         /*
 668          * Need a cursor.  Can't allocate until bb_level is filled in.
 669          */
 670         cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork);
 671         if (wasdel)
 672                 cur->bc_flags |= XFS_BTREE_BMBT_WASDEL;
 673         /*
 674          * Convert to a btree with two levels, one record in root.
 675          */
 676         ifp->if_format = XFS_DINODE_FMT_BTREE;
 677         memset(&args, 0, sizeof(args));
 678         args.tp = tp;
 679         args.mp = mp;
 680         xfs_rmap_ino_bmbt_owner(&args.oinfo, ip->i_ino, whichfork);
 681
 682         args.minlen = args.maxlen = args.prod = 1;
 683         args.wasdel = wasdel;
 684         *logflagsp = 0;
 685         error = xfs_alloc_vextent_start_ag(&args,
 686                                 XFS_INO_TO_FSB(mp, ip->i_ino));
 687         if (error)
 688                 goto out_root_realloc;
 689
 690         /*
 691          * Allocation can't fail, the space was reserved.
 692          */
 693         if (WARN_ON_ONCE(args.fsbno == NULLFSBLOCK)) {
 694                 error = -ENOSPC;
 695                 goto out_root_realloc;
 696         }
 697
 698         cur->bc_bmap.allocated++;
 699         ip->i_nblocks++;
 700         xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, 1L);
 701         error = xfs_trans_get_buf(tp, mp->m_ddev_targp,
 702                         XFS_FSB_TO_DADDR(mp, args.fsbno),
 703                         mp->m_bsize, 0, &abp);
 704         if (error)
 705                 goto out_unreserve_dquot;
 706
 707         /*
 708          * Fill in the child block.
 709          */
 710         ablock = XFS_BUF_TO_BLOCK(abp);
 711         xfs_bmbt_init_block(ip, ablock, abp, 0, 0);
 712
 713         for_each_xfs_iext(ifp, &icur, &rec) {
 714                 if (isnullstartblock(rec.br_startblock))
 715                         continue;
 716                 arp = XFS_BMBT_REC_ADDR(mp, ablock, 1 + cnt);
 717                 xfs_bmbt_disk_set_all(arp, &rec);
 718                 cnt++;
 719         }
 720         ASSERT(cnt == ifp->if_nextents);
 721         xfs_btree_set_numrecs(ablock, cnt);
 722
 723         /*
 724          * Fill in the root key and pointer.
 725          */
 726         kp = XFS_BMBT_KEY_ADDR(mp, block, 1);
 727         arp = XFS_BMBT_REC_ADDR(mp, ablock, 1);
 728         kp->br_startoff = cpu_to_be64(xfs_bmbt_disk_get_startoff(arp));
 729         pp = XFS_BMBT_PTR_ADDR(mp, block, 1, xfs_bmbt_get_maxrecs(cur,
 730                                                 be16_to_cpu(block->bb_level)));
 731         *pp = cpu_to_be64(args.fsbno);
 732
 733         /*
 734          * Do all this logging at the end so that
 735          * the root is at the right level.
 736          */
 737         xfs_btree_log_block(cur, abp, XFS_BB_ALL_BITS);
 738         xfs_btree_log_recs(cur, abp, 1, be16_to_cpu(ablock->bb_numrecs));
 739         ASSERT(*curp == NULL);
 740         *curp = cur;
 741         *logflagsp = XFS_ILOG_CORE | xfs_ilog_fbroot(whichfork);
 742         return 0;
 743
 744 out_unreserve_dquot:
 745         xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, -1L);
 746 out_root_realloc:
 747         xfs_iroot_realloc(ip, -1, whichfork);
 748         ifp->if_format = XFS_DINODE_FMT_EXTENTS;
 749         ASSERT(ifp->if_broot == NULL);
 750         xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
 751
 752         return error;
 753 }
 754
 755 /*
 756  * Convert a local file to an extents file.
 757  * This code is out of bounds for data forks of regular files,
 758  * since the file data needs to get logged so things will stay consistent.
 759  * (The bmap-level manipulations are ok, though).
 760  */
 761 void
 762 xfs_bmap_local_to_extents_empty(
 763         struct xfs_trans        *tp,
 764         struct xfs_inode        *ip,
 765         int                     whichfork)
 766 {
 767         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
 768
 769         ASSERT(whichfork != XFS_COW_FORK);
 770         ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL);
 771         ASSERT(ifp->if_bytes == 0);
 772         ASSERT(ifp->if_nextents == 0);
 773
 774         xfs_bmap_forkoff_reset(ip, whichfork);
 775         ifp->if_data = NULL;
 776         ifp->if_height = 0;
 777         ifp->if_format = XFS_DINODE_FMT_EXTENTS;
 778         xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
 779 }
 780
 781
 782 int                                     /* error */
 783 xfs_bmap_local_to_extents(
 784         xfs_trans_t     *tp,            /* transaction pointer */
 785         xfs_inode_t     *ip,            /* incore inode pointer */
 786         xfs_extlen_t    total,          /* total blocks needed by transaction */
 787         int             *logflagsp,     /* inode logging flags */
 788         int             whichfork,
 789         void            (*init_fn)(struct xfs_trans *tp,
 790                                    struct xfs_buf *bp,
 791                                    struct xfs_inode *ip,
 792                                    struct xfs_ifork *ifp, void *priv),
 793         void            *priv)
 794 {
 795         int             error = 0;
 796         int             flags;          /* logging flags returned */
 797         struct xfs_ifork *ifp;          /* inode fork pointer */
 798         xfs_alloc_arg_t args;           /* allocation arguments */
 799         struct xfs_buf  *bp;            /* buffer for extent block */
 800         struct xfs_bmbt_irec rec;
 801         struct xfs_iext_cursor icur;
 802
 803         /*
 804          * We don't want to deal with the case of keeping inode data inline yet.
 805          * So sending the data fork of a regular inode is invalid.
 806          */
 807         ASSERT(!(S_ISREG(VFS_I(ip)->i_mode) && whichfork == XFS_DATA_FORK));
 808         ifp = xfs_ifork_ptr(ip, whichfork);
 809         ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL);
 810
 811         if (!ifp->if_bytes) {
 812                 xfs_bmap_local_to_extents_empty(tp, ip, whichfork);
 813                 flags = XFS_ILOG_CORE;
 814                 goto done;
 815         }
 816
 817         flags = 0;
 818         error = 0;
 819         memset(&args, 0, sizeof(args));
 820         args.tp = tp;
 821         args.mp = ip->i_mount;
 822         args.total = total;
 823         args.minlen = args.maxlen = args.prod = 1;
 824         xfs_rmap_ino_owner(&args.oinfo, ip->i_ino, whichfork, 0);
 825
 826         /*
 827          * Allocate a block.  We know we need only one, since the
 828          * file currently fits in an inode.
 829          */
 830         args.total = total;
 831         args.minlen = args.maxlen = args.prod = 1;
 832         error = xfs_alloc_vextent_start_ag(&args,
 833                         XFS_INO_TO_FSB(args.mp, ip->i_ino));
 834         if (error)
 835                 goto done;
 836
 837         /* Can't fail, the space was reserved. */
 838         ASSERT(args.fsbno != NULLFSBLOCK);
 839         ASSERT(args.len == 1);
 840         error = xfs_trans_get_buf(tp, args.mp->m_ddev_targp,
 841                         XFS_FSB_TO_DADDR(args.mp, args.fsbno),
 842                         args.mp->m_bsize, 0, &bp);
 843         if (error)
 844                 goto done;
 845
 846         /*
 847          * Initialize the block, copy the data and log the remote buffer.
 848          *
 849          * The callout is responsible for logging because the remote format
 850          * might differ from the local format and thus we don't know how much to
 851          * log here. Note that init_fn must also set the buffer log item type
 852          * correctly.
 853          */
 854         init_fn(tp, bp, ip, ifp, priv);
 855
 856         /* account for the change in fork size */
 857         xfs_idata_realloc(ip, -ifp->if_bytes, whichfork);
 858         xfs_bmap_local_to_extents_empty(tp, ip, whichfork);
 859         flags |= XFS_ILOG_CORE;
 860
 861         ifp->if_data = NULL;
 862         ifp->if_height = 0;
 863
 864         rec.br_startoff = 0;
 865         rec.br_startblock = args.fsbno;
 866         rec.br_blockcount = 1;
 867         rec.br_state = XFS_EXT_NORM;
 868         xfs_iext_first(ifp, &icur);
 869         xfs_iext_insert(ip, &icur, &rec, 0);
 870
 871         ifp->if_nextents = 1;
 872         ip->i_nblocks = 1;
 873         xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, 1L);
 874         flags |= xfs_ilog_fext(whichfork);
 875
 876 done:
 877         *logflagsp = flags;
 878         return error;
 879 }
 880
 881 /*
 882  * Called from xfs_bmap_add_attrfork to handle btree format files.
 883  */
 884 STATIC int                                      /* error */
 885 xfs_bmap_add_attrfork_btree(
 886         xfs_trans_t             *tp,            /* transaction pointer */
 887         xfs_inode_t             *ip,            /* incore inode pointer */
 888         int                     *flags)         /* inode logging flags */
 889 {
 890         struct xfs_btree_block  *block = ip->i_df.if_broot;
 891         struct xfs_btree_cur    *cur;           /* btree cursor */
 892         int                     error;          /* error return value */
 893         xfs_mount_t             *mp;            /* file system mount struct */
 894         int                     stat;           /* newroot status */
 895
 896         mp = ip->i_mount;
 897
 898         if (XFS_BMAP_BMDR_SPACE(block) <= xfs_inode_data_fork_size(ip))
 899                 *flags |= XFS_ILOG_DBROOT;
 900         else {
 901                 cur = xfs_bmbt_init_cursor(mp, tp, ip, XFS_DATA_FORK);
 902                 error = xfs_bmbt_lookup_first(cur, &stat);
 903                 if (error)
 904                         goto error0;
 905                 /* must be at least one entry */
 906                 if (XFS_IS_CORRUPT(mp, stat != 1)) {
 907                         xfs_btree_mark_sick(cur);
 908                         error = -EFSCORRUPTED;
 909                         goto error0;
 910                 }
 911                 if ((error = xfs_btree_new_iroot(cur, flags, &stat)))
 912                         goto error0;
 913                 if (stat == 0) {
 914                         xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
 915                         return -ENOSPC;
 916                 }
 917                 cur->bc_bmap.allocated = 0;
 918                 xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
 919         }
 920         return 0;
 921 error0:
 922         xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
 923         return error;
 924 }
 925
 926 /*
 927  * Called from xfs_bmap_add_attrfork to handle extents format files.
 928  */
 929 STATIC int                                      /* error */
 930 xfs_bmap_add_attrfork_extents(
 931         struct xfs_trans        *tp,            /* transaction pointer */
 932         struct xfs_inode        *ip,            /* incore inode pointer */
 933         int                     *flags)         /* inode logging flags */
 934 {
 935         struct xfs_btree_cur    *cur;           /* bmap btree cursor */
 936         int                     error;          /* error return value */
 937
 938         if (ip->i_df.if_nextents * sizeof(struct xfs_bmbt_rec) <=
 939             xfs_inode_data_fork_size(ip))
 940                 return 0;
 941         cur = NULL;
 942         error = xfs_bmap_extents_to_btree(tp, ip, &cur, 0, flags,
 943                                           XFS_DATA_FORK);
 944         if (cur) {
 945                 cur->bc_bmap.allocated = 0;
 946                 xfs_btree_del_cursor(cur, error);
 947         }
 948         return error;
 949 }
 950
 951 /*
 952  * Called from xfs_bmap_add_attrfork to handle local format files. Each
 953  * different data fork content type needs a different callout to do the
 954  * conversion. Some are basic and only require special block initialisation
 955  * callouts for the data formating, others (directories) are so specialised they
 956  * handle everything themselves.
 957  *
 958  * XXX (dgc): investigate whether directory conversion can use the generic
 959  * formatting callout. It should be possible - it's just a very complex
 960  * formatter.
 961  */
 962 STATIC int                                      /* error */
 963 xfs_bmap_add_attrfork_local(
 964         struct xfs_trans        *tp,            /* transaction pointer */
 965         struct xfs_inode        *ip,            /* incore inode pointer */
 966         int                     *flags)         /* inode logging flags */
 967 {
 968         struct xfs_da_args      dargs;          /* args for dir/attr code */
 969
 970         if (ip->i_df.if_bytes <= xfs_inode_data_fork_size(ip))
 971                 return 0;
 972
 973         if (S_ISDIR(VFS_I(ip)->i_mode)) {
 974                 memset(&dargs, 0, sizeof(dargs));
 975                 dargs.geo = ip->i_mount->m_dir_geo;
 976                 dargs.dp = ip;
 977                 dargs.total = dargs.geo->fsbcount;
 978                 dargs.whichfork = XFS_DATA_FORK;
 979                 dargs.trans = tp;
 980                 dargs.owner = ip->i_ino;
 981                 return xfs_dir2_sf_to_block(&dargs);
 982         }
 983
 984         if (S_ISLNK(VFS_I(ip)->i_mode))
 985                 return xfs_bmap_local_to_extents(tp, ip, 1, flags,
 986                                 XFS_DATA_FORK, xfs_symlink_local_to_remote,
 987                                 NULL);
 988
 989         /* should only be called for types that support local format data */
 990         ASSERT(0);
 991         xfs_bmap_mark_sick(ip, XFS_ATTR_FORK);
 992         return -EFSCORRUPTED;
 993 }
 994
 995 /*
 996  * Set an inode attr fork offset based on the format of the data fork.
 997  */
 998 static int
 999 xfs_bmap_set_attrforkoff(
1000         struct xfs_inode        *ip,
1001         int                     size,
1002         int                     *version)
1003 {
1004         int                     default_size = xfs_default_attroffset(ip) >> 3;
1005
1006         switch (ip->i_df.if_format) {
1007         case XFS_DINODE_FMT_DEV:
1008                 ip->i_forkoff = default_size;
1009                 break;
1010         case XFS_DINODE_FMT_LOCAL:
1011         case XFS_DINODE_FMT_EXTENTS:
1012         case XFS_DINODE_FMT_BTREE:
1013                 ip->i_forkoff = xfs_attr_shortform_bytesfit(ip, size);
1014                 if (!ip->i_forkoff)
1015                         ip->i_forkoff = default_size;
1016                 else if (xfs_has_attr2(ip->i_mount) && version)
1017                         *version = 2;
1018                 break;
1019         default:
1020                 ASSERT(0);
1021                 return -EINVAL;
1022         }
1023
1024         return 0;
1025 }
1026
1027 /*
1028  * Convert inode from non-attributed to attributed.
1029  * Must not be in a transaction, ip must not be locked.
1030  */
1031 int                                             /* error code */
1032 xfs_bmap_add_attrfork(
1033         xfs_inode_t             *ip,            /* incore inode pointer */
1034         int                     size,           /* space new attribute needs */
1035         int                     rsvd)           /* xact may use reserved blks */
1036 {
1037         xfs_mount_t             *mp;            /* mount structure */
1038         xfs_trans_t             *tp;            /* transaction pointer */
1039         int                     blks;           /* space reservation */
1040         int                     version = 1;    /* superblock attr version */
1041         int                     logflags;       /* logging flags */
1042         int                     error;          /* error return value */
1043
1044         ASSERT(xfs_inode_has_attr_fork(ip) == 0);
1045
1046         mp = ip->i_mount;
1047         ASSERT(!XFS_NOT_DQATTACHED(mp, ip));
1048
1049         blks = XFS_ADDAFORK_SPACE_RES(mp);
1050
1051         error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_addafork, blks, 0,
1052                         rsvd, &tp);
1053         if (error)
1054                 return error;
1055         if (xfs_inode_has_attr_fork(ip))
1056                 goto trans_cancel;
1057
1058         xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
1059         error = xfs_bmap_set_attrforkoff(ip, size, &version);
1060         if (error)
1061                 goto trans_cancel;
1062
1063         xfs_ifork_init_attr(ip, XFS_DINODE_FMT_EXTENTS, 0);
1064         logflags = 0;
1065         switch (ip->i_df.if_format) {
1066         case XFS_DINODE_FMT_LOCAL:
1067                 error = xfs_bmap_add_attrfork_local(tp, ip, &logflags);
1068                 break;
1069         case XFS_DINODE_FMT_EXTENTS:
1070                 error = xfs_bmap_add_attrfork_extents(tp, ip, &logflags);
1071                 break;
1072         case XFS_DINODE_FMT_BTREE:
1073                 error = xfs_bmap_add_attrfork_btree(tp, ip, &logflags);
1074                 break;
1075         default:
1076                 error = 0;
1077                 break;
1078         }
1079         if (logflags)
1080                 xfs_trans_log_inode(tp, ip, logflags);
1081         if (error)
1082                 goto trans_cancel;
1083         if (!xfs_has_attr(mp) ||
1084            (!xfs_has_attr2(mp) && version == 2)) {
1085                 bool log_sb = false;
1086
1087                 spin_lock(&mp->m_sb_lock);
1088                 if (!xfs_has_attr(mp)) {
1089                         xfs_add_attr(mp);
1090                         log_sb = true;
1091                 }
1092                 if (!xfs_has_attr2(mp) && version == 2) {
1093                         xfs_add_attr2(mp);
1094                         log_sb = true;
1095                 }
1096                 spin_unlock(&mp->m_sb_lock);
1097                 if (log_sb)
1098                         xfs_log_sb(tp);
1099         }
1100
1101         error = xfs_trans_commit(tp);
1102         xfs_iunlock(ip, XFS_ILOCK_EXCL);
1103         return error;
1104
1105 trans_cancel:
1106         xfs_trans_cancel(tp);
1107         xfs_iunlock(ip, XFS_ILOCK_EXCL);
1108         return error;
1109 }
1110
1111 /*
1112  * Internal and external extent tree search functions.
1113  */
1114
1115 struct xfs_iread_state {
1116         struct xfs_iext_cursor  icur;
1117         xfs_extnum_t            loaded;
1118 };
1119
1120 int
1121 xfs_bmap_complain_bad_rec(
1122         struct xfs_inode                *ip,
1123         int                             whichfork,
1124         xfs_failaddr_t                  fa,
1125         const struct xfs_bmbt_irec      *irec)
1126 {
1127         struct xfs_mount                *mp = ip->i_mount;
1128         const char                      *forkname;
1129
1130         switch (whichfork) {
1131         case XFS_DATA_FORK:     forkname = "data"; break;
1132         case XFS_ATTR_FORK:     forkname = "attr"; break;
1133         case XFS_COW_FORK:      forkname = "CoW"; break;
1134         default:                forkname = "???"; break;
1135         }
1136
1137         xfs_warn(mp,
1138  "Bmap BTree record corruption in inode 0x%llx %s fork detected at %pS!",
1139                                 ip->i_ino, forkname, fa);
1140         xfs_warn(mp,
1141                 "Offset 0x%llx, start block 0x%llx, block count 0x%llx state 0x%x",
1142                 irec->br_startoff, irec->br_startblock, irec->br_blockcount,
1143                 irec->br_state);
1144
1145         return -EFSCORRUPTED;
1146 }
1147
1148 /* Stuff every bmbt record from this block into the incore extent map. */
1149 static int
1150 xfs_iread_bmbt_block(
1151         struct xfs_btree_cur    *cur,
1152         int                     level,
1153         void                    *priv)
1154 {
1155         struct xfs_iread_state  *ir = priv;
1156         struct xfs_mount        *mp = cur->bc_mp;
1157         struct xfs_inode        *ip = cur->bc_ino.ip;
1158         struct xfs_btree_block  *block;
1159         struct xfs_buf          *bp;
1160         struct xfs_bmbt_rec     *frp;
1161         xfs_extnum_t            num_recs;
1162         xfs_extnum_t            j;
1163         int                     whichfork = cur->bc_ino.whichfork;
1164         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
1165
1166         block = xfs_btree_get_block(cur, level, &bp);
1167
1168         /* Abort if we find more records than nextents. */
1169         num_recs = xfs_btree_get_numrecs(block);
1170         if (unlikely(ir->loaded + num_recs > ifp->if_nextents)) {
1171                 xfs_warn(ip->i_mount, "corrupt dinode %llu, (btree extents).",
1172                                 (unsigned long long)ip->i_ino);
1173                 xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, block,
1174                                 sizeof(*block), __this_address);
1175                 xfs_bmap_mark_sick(ip, whichfork);
1176                 return -EFSCORRUPTED;
1177         }
1178
1179         /* Copy records into the incore cache. */
1180         frp = XFS_BMBT_REC_ADDR(mp, block, 1);
1181         for (j = 0; j < num_recs; j++, frp++, ir->loaded++) {
1182                 struct xfs_bmbt_irec    new;
1183                 xfs_failaddr_t          fa;
1184
1185                 xfs_bmbt_disk_get_all(frp, &new);
1186                 fa = xfs_bmap_validate_extent(ip, whichfork, &new);
1187                 if (fa) {
1188                         xfs_inode_verifier_error(ip, -EFSCORRUPTED,
1189                                         "xfs_iread_extents(2)", frp,
1190                                         sizeof(*frp), fa);
1191                         xfs_bmap_mark_sick(ip, whichfork);
1192                         return xfs_bmap_complain_bad_rec(ip, whichfork, fa,
1193                                         &new);
1194                 }
1195                 xfs_iext_insert(ip, &ir->icur, &new,
1196                                 xfs_bmap_fork_to_state(whichfork));
1197                 trace_xfs_read_extent(ip, &ir->icur,
1198                                 xfs_bmap_fork_to_state(whichfork), _THIS_IP_);
1199                 xfs_iext_next(ifp, &ir->icur);
1200         }
1201
1202         return 0;
1203 }
1204
1205 /*
1206  * Read in extents from a btree-format inode.
1207  */
1208 int
1209 xfs_iread_extents(
1210         struct xfs_trans        *tp,
1211         struct xfs_inode        *ip,
1212         int                     whichfork)
1213 {
1214         struct xfs_iread_state  ir;
1215         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
1216         struct xfs_mount        *mp = ip->i_mount;
1217         struct xfs_btree_cur    *cur;
1218         int                     error;
1219
1220         if (!xfs_need_iread_extents(ifp))
1221                 return 0;
1222
1223         xfs_assert_ilocked(ip, XFS_ILOCK_EXCL);
1224
1225         ir.loaded = 0;
1226         xfs_iext_first(ifp, &ir.icur);
1227         cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork);
1228         error = xfs_btree_visit_blocks(cur, xfs_iread_bmbt_block,
1229                         XFS_BTREE_VISIT_RECORDS, &ir);
1230         xfs_btree_del_cursor(cur, error);
1231         if (error)
1232                 goto out;
1233
1234         if (XFS_IS_CORRUPT(mp, ir.loaded != ifp->if_nextents)) {
1235                 xfs_bmap_mark_sick(ip, whichfork);
1236                 error = -EFSCORRUPTED;
1237                 goto out;
1238         }
1239         ASSERT(ir.loaded == xfs_iext_count(ifp));
1240         /*
1241          * Use release semantics so that we can use acquire semantics in
1242          * xfs_need_iread_extents and be guaranteed to see a valid mapping tree
1243          * after that load.
1244          */
1245         smp_store_release(&ifp->if_needextents, 0);
1246         return 0;
1247 out:
1248         if (xfs_metadata_is_sick(error))
1249                 xfs_bmap_mark_sick(ip, whichfork);
1250         xfs_iext_destroy(ifp);
1251         return error;
1252 }
1253
1254 /*
1255  * Returns the relative block number of the first unused block(s) in the given
1256  * fork with at least "len" logically contiguous blocks free.  This is the
1257  * lowest-address hole if the fork has holes, else the first block past the end
1258  * of fork.  Return 0 if the fork is currently local (in-inode).
1259  */
1260 int                                             /* error */
1261 xfs_bmap_first_unused(
1262         struct xfs_trans        *tp,            /* transaction pointer */
1263         struct xfs_inode        *ip,            /* incore inode */
1264         xfs_extlen_t            len,            /* size of hole to find */
1265         xfs_fileoff_t           *first_unused,  /* unused block */
1266         int                     whichfork)      /* data or attr fork */
1267 {
1268         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
1269         struct xfs_bmbt_irec    got;
1270         struct xfs_iext_cursor  icur;
1271         xfs_fileoff_t           lastaddr = 0;
1272         xfs_fileoff_t           lowest, max;
1273         int                     error;
1274
1275         if (ifp->if_format == XFS_DINODE_FMT_LOCAL) {
1276                 *first_unused = 0;
1277                 return 0;
1278         }
1279
1280         ASSERT(xfs_ifork_has_extents(ifp));
1281
1282         error = xfs_iread_extents(tp, ip, whichfork);
1283         if (error)
1284                 return error;
1285
1286         lowest = max = *first_unused;
1287         for_each_xfs_iext(ifp, &icur, &got) {
1288                 /*
1289                  * See if the hole before this extent will work.
1290                  */
1291                 if (got.br_startoff >= lowest + len &&
1292                     got.br_startoff - max >= len)
1293                         break;
1294                 lastaddr = got.br_startoff + got.br_blockcount;
1295                 max = XFS_FILEOFF_MAX(lastaddr, lowest);
1296         }
1297
1298         *first_unused = max;
1299         return 0;
1300 }
1301
1302 /*
1303  * Returns the file-relative block number of the last block - 1 before
1304  * last_block (input value) in the file.
1305  * This is not based on i_size, it is based on the extent records.
1306  * Returns 0 for local files, as they do not have extent records.
1307  */
1308 int                                             /* error */
1309 xfs_bmap_last_before(
1310         struct xfs_trans        *tp,            /* transaction pointer */
1311         struct xfs_inode        *ip,            /* incore inode */
1312         xfs_fileoff_t           *last_block,    /* last block */
1313         int                     whichfork)      /* data or attr fork */
1314 {
1315         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
1316         struct xfs_bmbt_irec    got;
1317         struct xfs_iext_cursor  icur;
1318         int                     error;
1319
1320         switch (ifp->if_format) {
1321         case XFS_DINODE_FMT_LOCAL:
1322                 *last_block = 0;
1323                 return 0;
1324         case XFS_DINODE_FMT_BTREE:
1325         case XFS_DINODE_FMT_EXTENTS:
1326                 break;
1327         default:
1328                 ASSERT(0);
1329                 xfs_bmap_mark_sick(ip, whichfork);
1330                 return -EFSCORRUPTED;
1331         }
1332
1333         error = xfs_iread_extents(tp, ip, whichfork);
1334         if (error)
1335                 return error;
1336
1337         if (!xfs_iext_lookup_extent_before(ip, ifp, last_block, &icur, &got))
1338                 *last_block = 0;
1339         return 0;
1340 }
1341
1342 int
1343 xfs_bmap_last_extent(
1344         struct xfs_trans        *tp,
1345         struct xfs_inode        *ip,
1346         int                     whichfork,
1347         struct xfs_bmbt_irec    *rec,
1348         int                     *is_empty)
1349 {
1350         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
1351         struct xfs_iext_cursor  icur;
1352         int                     error;
1353
1354         error = xfs_iread_extents(tp, ip, whichfork);
1355         if (error)
1356                 return error;
1357
1358         xfs_iext_last(ifp, &icur);
1359         if (!xfs_iext_get_extent(ifp, &icur, rec))
1360                 *is_empty = 1;
1361         else
1362                 *is_empty = 0;
1363         return 0;
1364 }
1365
1366 /*
1367  * Check the last inode extent to determine whether this allocation will result
1368  * in blocks being allocated at the end of the file. When we allocate new data
1369  * blocks at the end of the file which do not start at the previous data block,
1370  * we will try to align the new blocks at stripe unit boundaries.
1371  *
1372  * Returns 1 in bma->aeof if the file (fork) is empty as any new write will be
1373  * at, or past the EOF.
1374  */
1375 STATIC int
1376 xfs_bmap_isaeof(
1377         struct xfs_bmalloca     *bma,
1378         int                     whichfork)
1379 {
1380         struct xfs_bmbt_irec    rec;
1381         int                     is_empty;
1382         int                     error;
1383
1384         bma->aeof = false;
1385         error = xfs_bmap_last_extent(NULL, bma->ip, whichfork, &rec,
1386                                      &is_empty);
1387         if (error)
1388                 return error;
1389
1390         if (is_empty) {
1391                 bma->aeof = true;
1392                 return 0;
1393         }
1394
1395         /*
1396          * Check if we are allocation or past the last extent, or at least into
1397          * the last delayed allocated extent.
1398          */
1399         bma->aeof = bma->offset >= rec.br_startoff + rec.br_blockcount ||
1400                 (bma->offset >= rec.br_startoff &&
1401                  isnullstartblock(rec.br_startblock));
1402         return 0;
1403 }
1404
1405 /*
1406  * Returns the file-relative block number of the first block past eof in
1407  * the file.  This is not based on i_size, it is based on the extent records.
1408  * Returns 0 for local files, as they do not have extent records.
1409  */
1410 int
1411 xfs_bmap_last_offset(
1412         struct xfs_inode        *ip,
1413         xfs_fileoff_t           *last_block,
1414         int                     whichfork)
1415 {
1416         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
1417         struct xfs_bmbt_irec    rec;
1418         int                     is_empty;
1419         int                     error;
1420
1421         *last_block = 0;
1422
1423         if (ifp->if_format == XFS_DINODE_FMT_LOCAL)
1424                 return 0;
1425
1426         if (XFS_IS_CORRUPT(ip->i_mount, !xfs_ifork_has_extents(ifp))) {
1427                 xfs_bmap_mark_sick(ip, whichfork);
1428                 return -EFSCORRUPTED;
1429         }
1430
1431         error = xfs_bmap_last_extent(NULL, ip, whichfork, &rec, &is_empty);
1432         if (error || is_empty)
1433                 return error;
1434
1435         *last_block = rec.br_startoff + rec.br_blockcount;
1436         return 0;
1437 }
1438
1439 /*
1440  * Extent tree manipulation functions used during allocation.
1441  */
1442
1443 /*
1444  * Convert a delayed allocation to a real allocation.
1445  */
1446 STATIC int                              /* error */
1447 xfs_bmap_add_extent_delay_real(
1448         struct xfs_bmalloca     *bma,
1449         int                     whichfork)
1450 {
1451         struct xfs_mount        *mp = bma->ip->i_mount;
1452         struct xfs_ifork        *ifp = xfs_ifork_ptr(bma->ip, whichfork);
1453         struct xfs_bmbt_irec    *new = &bma->got;
1454         int                     error;  /* error return value */
1455         int                     i;      /* temp state */
1456         xfs_fileoff_t           new_endoff;     /* end offset of new entry */
1457         xfs_bmbt_irec_t         r[3];   /* neighbor extent entries */
1458                                         /* left is 0, right is 1, prev is 2 */
1459         int                     rval=0; /* return value (logging flags) */
1460         uint32_t                state = xfs_bmap_fork_to_state(whichfork);
1461         xfs_filblks_t           da_new; /* new count del alloc blocks used */
1462         xfs_filblks_t           da_old; /* old count del alloc blocks used */
1463         xfs_filblks_t           temp=0; /* value for da_new calculations */
1464         int                     tmp_rval;       /* partial logging flags */
1465         struct xfs_bmbt_irec    old;
1466
1467         ASSERT(whichfork != XFS_ATTR_FORK);
1468         ASSERT(!isnullstartblock(new->br_startblock));
1469         ASSERT(!bma->cur || (bma->cur->bc_flags & XFS_BTREE_BMBT_WASDEL));
1470
1471         XFS_STATS_INC(mp, xs_add_exlist);
1472
1473 #define LEFT            r[0]
1474 #define RIGHT           r[1]
1475 #define PREV            r[2]
1476
1477         /*
1478          * Set up a bunch of variables to make the tests simpler.
1479          */
1480         xfs_iext_get_extent(ifp, &bma->icur, &PREV);
1481         new_endoff = new->br_startoff + new->br_blockcount;
1482         ASSERT(isnullstartblock(PREV.br_startblock));
1483         ASSERT(PREV.br_startoff <= new->br_startoff);
1484         ASSERT(PREV.br_startoff + PREV.br_blockcount >= new_endoff);
1485
1486         da_old = startblockval(PREV.br_startblock);
1487         da_new = 0;
1488
1489         /*
1490          * Set flags determining what part of the previous delayed allocation
1491          * extent is being replaced by a real allocation.
1492          */
1493         if (PREV.br_startoff == new->br_startoff)
1494                 state |= BMAP_LEFT_FILLING;
1495         if (PREV.br_startoff + PREV.br_blockcount == new_endoff)
1496                 state |= BMAP_RIGHT_FILLING;
1497
1498         /*
1499          * Check and set flags if this segment has a left neighbor.
1500          * Don't set contiguous if the combined extent would be too large.
1501          */
1502         if (xfs_iext_peek_prev_extent(ifp, &bma->icur, &LEFT)) {
1503                 state |= BMAP_LEFT_VALID;
1504                 if (isnullstartblock(LEFT.br_startblock))
1505                         state |= BMAP_LEFT_DELAY;
1506         }
1507
1508         if ((state & BMAP_LEFT_VALID) && !(state & BMAP_LEFT_DELAY) &&
1509             LEFT.br_startoff + LEFT.br_blockcount == new->br_startoff &&
1510             LEFT.br_startblock + LEFT.br_blockcount == new->br_startblock &&
1511             LEFT.br_state == new->br_state &&
1512             LEFT.br_blockcount + new->br_blockcount <= XFS_MAX_BMBT_EXTLEN)
1513                 state |= BMAP_LEFT_CONTIG;
1514
1515         /*
1516          * Check and set flags if this segment has a right neighbor.
1517          * Don't set contiguous if the combined extent would be too large.
1518          * Also check for all-three-contiguous being too large.
1519          */
1520         if (xfs_iext_peek_next_extent(ifp, &bma->icur, &RIGHT)) {
1521                 state |= BMAP_RIGHT_VALID;
1522                 if (isnullstartblock(RIGHT.br_startblock))
1523                         state |= BMAP_RIGHT_DELAY;
1524         }
1525
1526         if ((state & BMAP_RIGHT_VALID) && !(state & BMAP_RIGHT_DELAY) &&
1527             new_endoff == RIGHT.br_startoff &&
1528             new->br_startblock + new->br_blockcount == RIGHT.br_startblock &&
1529             new->br_state == RIGHT.br_state &&
1530             new->br_blockcount + RIGHT.br_blockcount <= XFS_MAX_BMBT_EXTLEN &&
1531             ((state & (BMAP_LEFT_CONTIG | BMAP_LEFT_FILLING |
1532                        BMAP_RIGHT_FILLING)) !=
1533                       (BMAP_LEFT_CONTIG | BMAP_LEFT_FILLING |
1534                        BMAP_RIGHT_FILLING) ||
1535              LEFT.br_blockcount + new->br_blockcount + RIGHT.br_blockcount
1536                         <= XFS_MAX_BMBT_EXTLEN))
1537                 state |= BMAP_RIGHT_CONTIG;
1538
1539         error = 0;
1540         /*
1541          * Switch out based on the FILLING and CONTIG state bits.
1542          */
1543         switch (state & (BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG |
1544                          BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG)) {
1545         case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG |
1546              BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG:
1547                 /*
1548                  * Filling in all of a previously delayed allocation extent.
1549                  * The left and right neighbors are both contiguous with new.
1550                  */
1551                 LEFT.br_blockcount += PREV.br_blockcount + RIGHT.br_blockcount;
1552
1553                 xfs_iext_remove(bma->ip, &bma->icur, state);
1554                 xfs_iext_remove(bma->ip, &bma->icur, state);
1555                 xfs_iext_prev(ifp, &bma->icur);
1556                 xfs_iext_update_extent(bma->ip, state, &bma->icur, &LEFT);
1557                 ifp->if_nextents--;
1558
1559                 if (bma->cur == NULL)
1560                         rval = XFS_ILOG_CORE | XFS_ILOG_DEXT;
1561                 else {
1562                         rval = XFS_ILOG_CORE;
1563                         error = xfs_bmbt_lookup_eq(bma->cur, &RIGHT, &i);
1564                         if (error)
1565                                 goto done;
1566                         if (XFS_IS_CORRUPT(mp, i != 1)) {
1567                                 xfs_btree_mark_sick(bma->cur);
1568                                 error = -EFSCORRUPTED;
1569                                 goto done;
1570                         }
1571                         error = xfs_btree_delete(bma->cur, &i);
1572                         if (error)
1573                                 goto done;
1574                         if (XFS_IS_CORRUPT(mp, i != 1)) {
1575                                 xfs_btree_mark_sick(bma->cur);
1576                                 error = -EFSCORRUPTED;
1577                                 goto done;
1578                         }
1579                         error = xfs_btree_decrement(bma->cur, 0, &i);
1580                         if (error)
1581                                 goto done;
1582                         if (XFS_IS_CORRUPT(mp, i != 1)) {
1583                                 xfs_btree_mark_sick(bma->cur);
1584                                 error = -EFSCORRUPTED;
1585                                 goto done;
1586                         }
1587                         error = xfs_bmbt_update(bma->cur, &LEFT);
1588                         if (error)
1589                                 goto done;
1590                 }
1591                 break;
1592
1593         case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG:
1594                 /*
1595                  * Filling in all of a previously delayed allocation extent.
1596                  * The left neighbor is contiguous, the right is not.
1597                  */
1598                 old = LEFT;
1599                 LEFT.br_blockcount += PREV.br_blockcount;
1600
1601                 xfs_iext_remove(bma->ip, &bma->icur, state);
1602                 xfs_iext_prev(ifp, &bma->icur);
1603                 xfs_iext_update_extent(bma->ip, state, &bma->icur, &LEFT);
1604
1605                 if (bma->cur == NULL)
1606                         rval = XFS_ILOG_DEXT;
1607                 else {
1608                         rval = 0;
1609                         error = xfs_bmbt_lookup_eq(bma->cur, &old, &i);
1610                         if (error)
1611                                 goto done;
1612                         if (XFS_IS_CORRUPT(mp, i != 1)) {
1613                                 xfs_btree_mark_sick(bma->cur);
1614                                 error = -EFSCORRUPTED;
1615                                 goto done;
1616                         }
1617                         error = xfs_bmbt_update(bma->cur, &LEFT);
1618                         if (error)
1619                                 goto done;
1620                 }
1621                 break;
1622
1623         case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG:
1624                 /*
1625                  * Filling in all of a previously delayed allocation extent.
1626                  * The right neighbor is contiguous, the left is not. Take care
1627                  * with delay -> unwritten extent allocation here because the
1628                  * delalloc record we are overwriting is always written.
1629                  */
1630                 PREV.br_startblock = new->br_startblock;
1631                 PREV.br_blockcount += RIGHT.br_blockcount;
1632                 PREV.br_state = new->br_state;
1633
1634                 xfs_iext_next(ifp, &bma->icur);
1635                 xfs_iext_remove(bma->ip, &bma->icur, state);
1636                 xfs_iext_prev(ifp, &bma->icur);
1637                 xfs_iext_update_extent(bma->ip, state, &bma->icur, &PREV);
1638
1639                 if (bma->cur == NULL)
1640                         rval = XFS_ILOG_DEXT;
1641                 else {
1642                         rval = 0;
1643                         error = xfs_bmbt_lookup_eq(bma->cur, &RIGHT, &i);
1644                         if (error)
1645                                 goto done;
1646                         if (XFS_IS_CORRUPT(mp, i != 1)) {
1647                                 xfs_btree_mark_sick(bma->cur);
1648                                 error = -EFSCORRUPTED;
1649                                 goto done;
1650                         }
1651                         error = xfs_bmbt_update(bma->cur, &PREV);
1652                         if (error)
1653                                 goto done;
1654                 }
1655                 break;
1656
1657         case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING:
1658                 /*
1659                  * Filling in all of a previously delayed allocation extent.
1660                  * Neither the left nor right neighbors are contiguous with
1661                  * the new one.
1662                  */
1663                 PREV.br_startblock = new->br_startblock;
1664                 PREV.br_state = new->br_state;
1665                 xfs_iext_update_extent(bma->ip, state, &bma->icur, &PREV);
1666                 ifp->if_nextents++;
1667
1668                 if (bma->cur == NULL)
1669                         rval = XFS_ILOG_CORE | XFS_ILOG_DEXT;
1670                 else {
1671                         rval = XFS_ILOG_CORE;
1672                         error = xfs_bmbt_lookup_eq(bma->cur, new, &i);
1673                         if (error)
1674                                 goto done;
1675                         if (XFS_IS_CORRUPT(mp, i != 0)) {
1676                                 xfs_btree_mark_sick(bma->cur);
1677                                 error = -EFSCORRUPTED;
1678                                 goto done;
1679                         }
1680                         error = xfs_btree_insert(bma->cur, &i);
1681                         if (error)
1682                                 goto done;
1683                         if (XFS_IS_CORRUPT(mp, i != 1)) {
1684                                 xfs_btree_mark_sick(bma->cur);
1685                                 error = -EFSCORRUPTED;
1686                                 goto done;
1687                         }
1688                 }
1689                 break;
1690
1691         case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG:
1692                 /*
1693                  * Filling in the first part of a previous delayed allocation.
1694                  * The left neighbor is contiguous.
1695                  */
1696                 old = LEFT;
1697                 temp = PREV.br_blockcount - new->br_blockcount;
1698                 da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(bma->ip, temp),
1699                                 startblockval(PREV.br_startblock));
1700
1701                 LEFT.br_blockcount += new->br_blockcount;
1702
1703                 PREV.br_blockcount = temp;
1704                 PREV.br_startoff += new->br_blockcount;
1705                 PREV.br_startblock = nullstartblock(da_new);
1706
1707                 xfs_iext_update_extent(bma->ip, state, &bma->icur, &PREV);
1708                 xfs_iext_prev(ifp, &bma->icur);
1709                 xfs_iext_update_extent(bma->ip, state, &bma->icur, &LEFT);
1710
1711                 if (bma->cur == NULL)
1712                         rval = XFS_ILOG_DEXT;
1713                 else {
1714                         rval = 0;
1715                         error = xfs_bmbt_lookup_eq(bma->cur, &old, &i);
1716                         if (error)
1717                                 goto done;
1718                         if (XFS_IS_CORRUPT(mp, i != 1)) {
1719                                 xfs_btree_mark_sick(bma->cur);
1720                                 error = -EFSCORRUPTED;
1721                                 goto done;
1722                         }
1723                         error = xfs_bmbt_update(bma->cur, &LEFT);
1724                         if (error)
1725                                 goto done;
1726                 }
1727                 break;
1728
1729         case BMAP_LEFT_FILLING:
1730                 /*
1731                  * Filling in the first part of a previous delayed allocation.
1732                  * The left neighbor is not contiguous.
1733                  */
1734                 xfs_iext_update_extent(bma->ip, state, &bma->icur, new);
1735                 ifp->if_nextents++;
1736
1737                 if (bma->cur == NULL)
1738                         rval = XFS_ILOG_CORE | XFS_ILOG_DEXT;
1739                 else {
1740                         rval = XFS_ILOG_CORE;
1741                         error = xfs_bmbt_lookup_eq(bma->cur, new, &i);
1742                         if (error)
1743                                 goto done;
1744                         if (XFS_IS_CORRUPT(mp, i != 0)) {
1745                                 xfs_btree_mark_sick(bma->cur);
1746                                 error = -EFSCORRUPTED;
1747                                 goto done;
1748                         }
1749                         error = xfs_btree_insert(bma->cur, &i);
1750                         if (error)
1751                                 goto done;
1752                         if (XFS_IS_CORRUPT(mp, i != 1)) {
1753                                 xfs_btree_mark_sick(bma->cur);
1754                                 error = -EFSCORRUPTED;
1755                                 goto done;
1756                         }
1757                 }
1758
1759                 if (xfs_bmap_needs_btree(bma->ip, whichfork)) {
1760                         error = xfs_bmap_extents_to_btree(bma->tp, bma->ip,
1761                                         &bma->cur, 1, &tmp_rval, whichfork);
1762                         rval |= tmp_rval;
1763                         if (error)
1764                                 goto done;
1765                 }
1766
1767                 temp = PREV.br_blockcount - new->br_blockcount;
1768                 da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(bma->ip, temp),
1769                         startblockval(PREV.br_startblock) -
1770                         (bma->cur ? bma->cur->bc_bmap.allocated : 0));
1771
1772                 PREV.br_startoff = new_endoff;
1773                 PREV.br_blockcount = temp;
1774                 PREV.br_startblock = nullstartblock(da_new);
1775                 xfs_iext_next(ifp, &bma->icur);
1776                 xfs_iext_insert(bma->ip, &bma->icur, &PREV, state);
1777                 xfs_iext_prev(ifp, &bma->icur);
1778                 break;
1779
1780         case BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG:
1781                 /*
1782                  * Filling in the last part of a previous delayed allocation.
1783                  * The right neighbor is contiguous with the new allocation.
1784                  */
1785                 old = RIGHT;
1786                 RIGHT.br_startoff = new->br_startoff;
1787                 RIGHT.br_startblock = new->br_startblock;
1788                 RIGHT.br_blockcount += new->br_blockcount;
1789
1790                 if (bma->cur == NULL)
1791                         rval = XFS_ILOG_DEXT;
1792                 else {
1793                         rval = 0;
1794                         error = xfs_bmbt_lookup_eq(bma->cur, &old, &i);
1795                         if (error)
1796                                 goto done;
1797                         if (XFS_IS_CORRUPT(mp, i != 1)) {
1798                                 xfs_btree_mark_sick(bma->cur);
1799                                 error = -EFSCORRUPTED;
1800                                 goto done;
1801                         }
1802                         error = xfs_bmbt_update(bma->cur, &RIGHT);
1803                         if (error)
1804                                 goto done;
1805                 }
1806
1807                 temp = PREV.br_blockcount - new->br_blockcount;
1808                 da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(bma->ip, temp),
1809                         startblockval(PREV.br_startblock));
1810
1811                 PREV.br_blockcount = temp;
1812                 PREV.br_startblock = nullstartblock(da_new);
1813
1814                 xfs_iext_update_extent(bma->ip, state, &bma->icur, &PREV);
1815                 xfs_iext_next(ifp, &bma->icur);
1816                 xfs_iext_update_extent(bma->ip, state, &bma->icur, &RIGHT);
1817                 break;
1818
1819         case BMAP_RIGHT_FILLING:
1820                 /*
1821                  * Filling in the last part of a previous delayed allocation.
1822                  * The right neighbor is not contiguous.
1823                  */
1824                 xfs_iext_update_extent(bma->ip, state, &bma->icur, new);
1825                 ifp->if_nextents++;
1826
1827                 if (bma->cur == NULL)
1828                         rval = XFS_ILOG_CORE | XFS_ILOG_DEXT;
1829                 else {
1830                         rval = XFS_ILOG_CORE;
1831                         error = xfs_bmbt_lookup_eq(bma->cur, new, &i);
1832                         if (error)
1833                                 goto done;
1834                         if (XFS_IS_CORRUPT(mp, i != 0)) {
1835                                 xfs_btree_mark_sick(bma->cur);
1836                                 error = -EFSCORRUPTED;
1837                                 goto done;
1838                         }
1839                         error = xfs_btree_insert(bma->cur, &i);
1840                         if (error)
1841                                 goto done;
1842                         if (XFS_IS_CORRUPT(mp, i != 1)) {
1843                                 xfs_btree_mark_sick(bma->cur);
1844                                 error = -EFSCORRUPTED;
1845                                 goto done;
1846                         }
1847                 }
1848
1849                 if (xfs_bmap_needs_btree(bma->ip, whichfork)) {
1850                         error = xfs_bmap_extents_to_btree(bma->tp, bma->ip,
1851                                 &bma->cur, 1, &tmp_rval, whichfork);
1852                         rval |= tmp_rval;
1853                         if (error)
1854                                 goto done;
1855                 }
1856
1857                 temp = PREV.br_blockcount - new->br_blockcount;
1858                 da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(bma->ip, temp),
1859                         startblockval(PREV.br_startblock) -
1860                         (bma->cur ? bma->cur->bc_bmap.allocated : 0));
1861
1862                 PREV.br_startblock = nullstartblock(da_new);
1863                 PREV.br_blockcount = temp;
1864                 xfs_iext_insert(bma->ip, &bma->icur, &PREV, state);
1865                 xfs_iext_next(ifp, &bma->icur);
1866                 break;
1867
1868         case 0:
1869                 /*
1870                  * Filling in the middle part of a previous delayed allocation.
1871                  * Contiguity is impossible here.
1872                  * This case is avoided almost all the time.
1873                  *
1874                  * We start with a delayed allocation:
1875                  *
1876                  * +ddddddddddddddddddddddddddddddddddddddddddddddddddddddd+
1877                  *  PREV @ idx
1878                  *
1879                  * and we are allocating:
1880                  *                     +rrrrrrrrrrrrrrrrr+
1881                  *                            new
1882                  *
1883                  * and we set it up for insertion as:
1884                  * +ddddddddddddddddddd+rrrrrrrrrrrrrrrrr+ddddddddddddddddd+
1885                  *                            new
1886                  *  PREV @ idx          LEFT              RIGHT
1887                  *                      inserted at idx + 1
1888                  */
1889                 old = PREV;
1890
1891                 /* LEFT is the new middle */
1892                 LEFT = *new;
1893
1894                 /* RIGHT is the new right */
1895                 RIGHT.br_state = PREV.br_state;
1896                 RIGHT.br_startoff = new_endoff;
1897                 RIGHT.br_blockcount =
1898                         PREV.br_startoff + PREV.br_blockcount - new_endoff;
1899                 RIGHT.br_startblock =
1900                         nullstartblock(xfs_bmap_worst_indlen(bma->ip,
1901                                         RIGHT.br_blockcount));
1902
1903                 /* truncate PREV */
1904                 PREV.br_blockcount = new->br_startoff - PREV.br_startoff;
1905                 PREV.br_startblock =
1906                         nullstartblock(xfs_bmap_worst_indlen(bma->ip,
1907                                         PREV.br_blockcount));
1908                 xfs_iext_update_extent(bma->ip, state, &bma->icur, &PREV);
1909
1910                 xfs_iext_next(ifp, &bma->icur);
1911                 xfs_iext_insert(bma->ip, &bma->icur, &RIGHT, state);
1912                 xfs_iext_insert(bma->ip, &bma->icur, &LEFT, state);
1913                 ifp->if_nextents++;
1914
1915                 if (bma->cur == NULL)
1916                         rval = XFS_ILOG_CORE | XFS_ILOG_DEXT;
1917                 else {
1918                         rval = XFS_ILOG_CORE;
1919                         error = xfs_bmbt_lookup_eq(bma->cur, new, &i);
1920                         if (error)
1921                                 goto done;
1922                         if (XFS_IS_CORRUPT(mp, i != 0)) {
1923                                 xfs_btree_mark_sick(bma->cur);
1924                                 error = -EFSCORRUPTED;
1925                                 goto done;
1926                         }
1927                         error = xfs_btree_insert(bma->cur, &i);
1928                         if (error)
1929                                 goto done;
1930                         if (XFS_IS_CORRUPT(mp, i != 1)) {
1931                                 xfs_btree_mark_sick(bma->cur);
1932                                 error = -EFSCORRUPTED;
1933                                 goto done;
1934                         }
1935                 }
1936
1937                 if (xfs_bmap_needs_btree(bma->ip, whichfork)) {
1938                         error = xfs_bmap_extents_to_btree(bma->tp, bma->ip,
1939                                         &bma->cur, 1, &tmp_rval, whichfork);
1940                         rval |= tmp_rval;
1941                         if (error)
1942                                 goto done;
1943                 }
1944
1945                 da_new = startblockval(PREV.br_startblock) +
1946                          startblockval(RIGHT.br_startblock);
1947                 break;
1948
1949         case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG:
1950         case BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG:
1951         case BMAP_LEFT_FILLING | BMAP_RIGHT_CONTIG:
1952         case BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG:
1953         case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG:
1954         case BMAP_LEFT_CONTIG:
1955         case BMAP_RIGHT_CONTIG:
1956                 /*
1957                  * These cases are all impossible.
1958                  */
1959                 ASSERT(0);
1960         }
1961
1962         /* add reverse mapping unless caller opted out */
1963         if (!(bma->flags & XFS_BMAPI_NORMAP))
1964                 xfs_rmap_map_extent(bma->tp, bma->ip, whichfork, new);
1965
1966         /* convert to a btree if necessary */
1967         if (xfs_bmap_needs_btree(bma->ip, whichfork)) {
1968                 int     tmp_logflags;   /* partial log flag return val */
1969
1970                 ASSERT(bma->cur == NULL);
1971                 error = xfs_bmap_extents_to_btree(bma->tp, bma->ip,
1972                                 &bma->cur, da_old > 0, &tmp_logflags,
1973                                 whichfork);
1974                 bma->logflags |= tmp_logflags;
1975                 if (error)
1976                         goto done;
1977         }
1978
1979         if (da_new != da_old)
1980                 xfs_mod_delalloc(bma->ip, 0, (int64_t)da_new - da_old);
1981
1982         if (bma->cur) {
1983                 da_new += bma->cur->bc_bmap.allocated;
1984                 bma->cur->bc_bmap.allocated = 0;
1985         }
1986
1987         /* adjust for changes in reserved delayed indirect blocks */
1988         if (da_new < da_old) {
1989                 xfs_add_fdblocks(mp, da_old - da_new);
1990         } else if (da_new > da_old) {
1991                 ASSERT(state == 0);
1992                 error = xfs_dec_fdblocks(mp, da_new - da_old, false);
1993         }
1994
1995         xfs_bmap_check_leaf_extents(bma->cur, bma->ip, whichfork);
1996 done:
1997         if (whichfork != XFS_COW_FORK)
1998                 bma->logflags |= rval;
1999         return error;
2000 #undef  LEFT
2001 #undef  RIGHT
2002 #undef  PREV
2003 }
2004
2005 /*
2006  * Convert an unwritten allocation to a real allocation or vice versa.
2007  */
2008 int                                     /* error */
2009 xfs_bmap_add_extent_unwritten_real(
2010         struct xfs_trans        *tp,
2011         xfs_inode_t             *ip,    /* incore inode pointer */
2012         int                     whichfork,
2013         struct xfs_iext_cursor  *icur,
2014         struct xfs_btree_cur    **curp, /* if *curp is null, not a btree */
2015         xfs_bmbt_irec_t         *new,   /* new data to add to file extents */
2016         int                     *logflagsp) /* inode logging flags */
2017 {
2018         struct xfs_btree_cur    *cur;   /* btree cursor */
2019         int                     error;  /* error return value */
2020         int                     i;      /* temp state */
2021         struct xfs_ifork        *ifp;   /* inode fork pointer */
2022         xfs_fileoff_t           new_endoff;     /* end offset of new entry */
2023         xfs_bmbt_irec_t         r[3];   /* neighbor extent entries */
2024                                         /* left is 0, right is 1, prev is 2 */
2025         int                     rval=0; /* return value (logging flags) */
2026         uint32_t                state = xfs_bmap_fork_to_state(whichfork);
2027         struct xfs_mount        *mp = ip->i_mount;
2028         struct xfs_bmbt_irec    old;
2029
2030         *logflagsp = 0;
2031
2032         cur = *curp;
2033         ifp = xfs_ifork_ptr(ip, whichfork);
2034
2035         ASSERT(!isnullstartblock(new->br_startblock));
2036
2037         XFS_STATS_INC(mp, xs_add_exlist);
2038
2039 #define LEFT            r[0]
2040 #define RIGHT           r[1]
2041 #define PREV            r[2]
2042
2043         /*
2044          * Set up a bunch of variables to make the tests simpler.
2045          */
2046         error = 0;
2047         xfs_iext_get_extent(ifp, icur, &PREV);
2048         ASSERT(new->br_state != PREV.br_state);
2049         new_endoff = new->br_startoff + new->br_blockcount;
2050         ASSERT(PREV.br_startoff <= new->br_startoff);
2051         ASSERT(PREV.br_startoff + PREV.br_blockcount >= new_endoff);
2052
2053         /*
2054          * Set flags determining what part of the previous oldext allocation
2055          * extent is being replaced by a newext allocation.
2056          */
2057         if (PREV.br_startoff == new->br_startoff)
2058                 state |= BMAP_LEFT_FILLING;
2059         if (PREV.br_startoff + PREV.br_blockcount == new_endoff)
2060                 state |= BMAP_RIGHT_FILLING;
2061
2062         /*
2063          * Check and set flags if this segment has a left neighbor.
2064          * Don't set contiguous if the combined extent would be too large.
2065          */
2066         if (xfs_iext_peek_prev_extent(ifp, icur, &LEFT)) {
2067                 state |= BMAP_LEFT_VALID;
2068                 if (isnullstartblock(LEFT.br_startblock))
2069                         state |= BMAP_LEFT_DELAY;
2070         }
2071
2072         if ((state & BMAP_LEFT_VALID) && !(state & BMAP_LEFT_DELAY) &&
2073             LEFT.br_startoff + LEFT.br_blockcount == new->br_startoff &&
2074             LEFT.br_startblock + LEFT.br_blockcount == new->br_startblock &&
2075             LEFT.br_state == new->br_state &&
2076             LEFT.br_blockcount + new->br_blockcount <= XFS_MAX_BMBT_EXTLEN)
2077                 state |= BMAP_LEFT_CONTIG;
2078
2079         /*
2080          * Check and set flags if this segment has a right neighbor.
2081          * Don't set contiguous if the combined extent would be too large.
2082          * Also check for all-three-contiguous being too large.
2083          */
2084         if (xfs_iext_peek_next_extent(ifp, icur, &RIGHT)) {
2085                 state |= BMAP_RIGHT_VALID;
2086                 if (isnullstartblock(RIGHT.br_startblock))
2087                         state |= BMAP_RIGHT_DELAY;
2088         }
2089
2090         if ((state & BMAP_RIGHT_VALID) && !(state & BMAP_RIGHT_DELAY) &&
2091             new_endoff == RIGHT.br_startoff &&
2092             new->br_startblock + new->br_blockcount == RIGHT.br_startblock &&
2093             new->br_state == RIGHT.br_state &&
2094             new->br_blockcount + RIGHT.br_blockcount <= XFS_MAX_BMBT_EXTLEN &&
2095             ((state & (BMAP_LEFT_CONTIG | BMAP_LEFT_FILLING |
2096                        BMAP_RIGHT_FILLING)) !=
2097                       (BMAP_LEFT_CONTIG | BMAP_LEFT_FILLING |
2098                        BMAP_RIGHT_FILLING) ||
2099              LEFT.br_blockcount + new->br_blockcount + RIGHT.br_blockcount
2100                         <= XFS_MAX_BMBT_EXTLEN))
2101                 state |= BMAP_RIGHT_CONTIG;
2102
2103         /*
2104          * Switch out based on the FILLING and CONTIG state bits.
2105          */
2106         switch (state & (BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG |
2107                          BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG)) {
2108         case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG |
2109              BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG:
2110                 /*
2111                  * Setting all of a previous oldext extent to newext.
2112                  * The left and right neighbors are both contiguous with new.
2113                  */
2114                 LEFT.br_blockcount += PREV.br_blockcount + RIGHT.br_blockcount;
2115
2116                 xfs_iext_remove(ip, icur, state);
2117                 xfs_iext_remove(ip, icur, state);
2118                 xfs_iext_prev(ifp, icur);
2119                 xfs_iext_update_extent(ip, state, icur, &LEFT);
2120                 ifp->if_nextents -= 2;
2121                 if (cur == NULL)
2122                         rval = XFS_ILOG_CORE | XFS_ILOG_DEXT;
2123                 else {
2124                         rval = XFS_ILOG_CORE;
2125                         error = xfs_bmbt_lookup_eq(cur, &RIGHT, &i);
2126                         if (error)
2127                                 goto done;
2128                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2129                                 xfs_btree_mark_sick(cur);
2130                                 error = -EFSCORRUPTED;
2131                                 goto done;
2132                         }
2133                         if ((error = xfs_btree_delete(cur, &i)))
2134                                 goto done;
2135                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2136                                 xfs_btree_mark_sick(cur);
2137                                 error = -EFSCORRUPTED;
2138                                 goto done;
2139                         }
2140                         if ((error = xfs_btree_decrement(cur, 0, &i)))
2141                                 goto done;
2142                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2143                                 xfs_btree_mark_sick(cur);
2144                                 error = -EFSCORRUPTED;
2145                                 goto done;
2146                         }
2147                         if ((error = xfs_btree_delete(cur, &i)))
2148                                 goto done;
2149                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2150                                 xfs_btree_mark_sick(cur);
2151                                 error = -EFSCORRUPTED;
2152                                 goto done;
2153                         }
2154                         if ((error = xfs_btree_decrement(cur, 0, &i)))
2155                                 goto done;
2156                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2157                                 xfs_btree_mark_sick(cur);
2158                                 error = -EFSCORRUPTED;
2159                                 goto done;
2160                         }
2161                         error = xfs_bmbt_update(cur, &LEFT);
2162                         if (error)
2163                                 goto done;
2164                 }
2165                 break;
2166
2167         case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG:
2168                 /*
2169                  * Setting all of a previous oldext extent to newext.
2170                  * The left neighbor is contiguous, the right is not.
2171                  */
2172                 LEFT.br_blockcount += PREV.br_blockcount;
2173
2174                 xfs_iext_remove(ip, icur, state);
2175                 xfs_iext_prev(ifp, icur);
2176                 xfs_iext_update_extent(ip, state, icur, &LEFT);
2177                 ifp->if_nextents--;
2178                 if (cur == NULL)
2179                         rval = XFS_ILOG_CORE | XFS_ILOG_DEXT;
2180                 else {
2181                         rval = XFS_ILOG_CORE;
2182                         error = xfs_bmbt_lookup_eq(cur, &PREV, &i);
2183                         if (error)
2184                                 goto done;
2185                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2186                                 xfs_btree_mark_sick(cur);
2187                                 error = -EFSCORRUPTED;
2188                                 goto done;
2189                         }
2190                         if ((error = xfs_btree_delete(cur, &i)))
2191                                 goto done;
2192                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2193                                 xfs_btree_mark_sick(cur);
2194                                 error = -EFSCORRUPTED;
2195                                 goto done;
2196                         }
2197                         if ((error = xfs_btree_decrement(cur, 0, &i)))
2198                                 goto done;
2199                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2200                                 xfs_btree_mark_sick(cur);
2201                                 error = -EFSCORRUPTED;
2202                                 goto done;
2203                         }
2204                         error = xfs_bmbt_update(cur, &LEFT);
2205                         if (error)
2206                                 goto done;
2207                 }
2208                 break;
2209
2210         case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG:
2211                 /*
2212                  * Setting all of a previous oldext extent to newext.
2213                  * The right neighbor is contiguous, the left is not.
2214                  */
2215                 PREV.br_blockcount += RIGHT.br_blockcount;
2216                 PREV.br_state = new->br_state;
2217
2218                 xfs_iext_next(ifp, icur);
2219                 xfs_iext_remove(ip, icur, state);
2220                 xfs_iext_prev(ifp, icur);
2221                 xfs_iext_update_extent(ip, state, icur, &PREV);
2222                 ifp->if_nextents--;
2223
2224                 if (cur == NULL)
2225                         rval = XFS_ILOG_CORE | XFS_ILOG_DEXT;
2226                 else {
2227                         rval = XFS_ILOG_CORE;
2228                         error = xfs_bmbt_lookup_eq(cur, &RIGHT, &i);
2229                         if (error)
2230                                 goto done;
2231                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2232                                 xfs_btree_mark_sick(cur);
2233                                 error = -EFSCORRUPTED;
2234                                 goto done;
2235                         }
2236                         if ((error = xfs_btree_delete(cur, &i)))
2237                                 goto done;
2238                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2239                                 xfs_btree_mark_sick(cur);
2240                                 error = -EFSCORRUPTED;
2241                                 goto done;
2242                         }
2243                         if ((error = xfs_btree_decrement(cur, 0, &i)))
2244                                 goto done;
2245                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2246                                 xfs_btree_mark_sick(cur);
2247                                 error = -EFSCORRUPTED;
2248                                 goto done;
2249                         }
2250                         error = xfs_bmbt_update(cur, &PREV);
2251                         if (error)
2252                                 goto done;
2253                 }
2254                 break;
2255
2256         case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING:
2257                 /*
2258                  * Setting all of a previous oldext extent to newext.
2259                  * Neither the left nor right neighbors are contiguous with
2260                  * the new one.
2261                  */
2262                 PREV.br_state = new->br_state;
2263                 xfs_iext_update_extent(ip, state, icur, &PREV);
2264
2265                 if (cur == NULL)
2266                         rval = XFS_ILOG_DEXT;
2267                 else {
2268                         rval = 0;
2269                         error = xfs_bmbt_lookup_eq(cur, new, &i);
2270                         if (error)
2271                                 goto done;
2272                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2273                                 xfs_btree_mark_sick(cur);
2274                                 error = -EFSCORRUPTED;
2275                                 goto done;
2276                         }
2277                         error = xfs_bmbt_update(cur, &PREV);
2278                         if (error)
2279                                 goto done;
2280                 }
2281                 break;
2282
2283         case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG:
2284                 /*
2285                  * Setting the first part of a previous oldext extent to newext.
2286                  * The left neighbor is contiguous.
2287                  */
2288                 LEFT.br_blockcount += new->br_blockcount;
2289
2290                 old = PREV;
2291                 PREV.br_startoff += new->br_blockcount;
2292                 PREV.br_startblock += new->br_blockcount;
2293                 PREV.br_blockcount -= new->br_blockcount;
2294
2295                 xfs_iext_update_extent(ip, state, icur, &PREV);
2296                 xfs_iext_prev(ifp, icur);
2297                 xfs_iext_update_extent(ip, state, icur, &LEFT);
2298
2299                 if (cur == NULL)
2300                         rval = XFS_ILOG_DEXT;
2301                 else {
2302                         rval = 0;
2303                         error = xfs_bmbt_lookup_eq(cur, &old, &i);
2304                         if (error)
2305                                 goto done;
2306                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2307                                 xfs_btree_mark_sick(cur);
2308                                 error = -EFSCORRUPTED;
2309                                 goto done;
2310                         }
2311                         error = xfs_bmbt_update(cur, &PREV);
2312                         if (error)
2313                                 goto done;
2314                         error = xfs_btree_decrement(cur, 0, &i);
2315                         if (error)
2316                                 goto done;
2317                         error = xfs_bmbt_update(cur, &LEFT);
2318                         if (error)
2319                                 goto done;
2320                 }
2321                 break;
2322
2323         case BMAP_LEFT_FILLING:
2324                 /*
2325                  * Setting the first part of a previous oldext extent to newext.
2326                  * The left neighbor is not contiguous.
2327                  */
2328                 old = PREV;
2329                 PREV.br_startoff += new->br_blockcount;
2330                 PREV.br_startblock += new->br_blockcount;
2331                 PREV.br_blockcount -= new->br_blockcount;
2332
2333                 xfs_iext_update_extent(ip, state, icur, &PREV);
2334                 xfs_iext_insert(ip, icur, new, state);
2335                 ifp->if_nextents++;
2336
2337                 if (cur == NULL)
2338                         rval = XFS_ILOG_CORE | XFS_ILOG_DEXT;
2339                 else {
2340                         rval = XFS_ILOG_CORE;
2341                         error = xfs_bmbt_lookup_eq(cur, &old, &i);
2342                         if (error)
2343                                 goto done;
2344                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2345                                 xfs_btree_mark_sick(cur);
2346                                 error = -EFSCORRUPTED;
2347                                 goto done;
2348                         }
2349                         error = xfs_bmbt_update(cur, &PREV);
2350                         if (error)
2351                                 goto done;
2352                         cur->bc_rec.b = *new;
2353                         if ((error = xfs_btree_insert(cur, &i)))
2354                                 goto done;
2355                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2356                                 xfs_btree_mark_sick(cur);
2357                                 error = -EFSCORRUPTED;
2358                                 goto done;
2359                         }
2360                 }
2361                 break;
2362
2363         case BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG:
2364                 /*
2365                  * Setting the last part of a previous oldext extent to newext.
2366                  * The right neighbor is contiguous with the new allocation.
2367                  */
2368                 old = PREV;
2369                 PREV.br_blockcount -= new->br_blockcount;
2370
2371                 RIGHT.br_startoff = new->br_startoff;
2372                 RIGHT.br_startblock = new->br_startblock;
2373                 RIGHT.br_blockcount += new->br_blockcount;
2374
2375                 xfs_iext_update_extent(ip, state, icur, &PREV);
2376                 xfs_iext_next(ifp, icur);
2377                 xfs_iext_update_extent(ip, state, icur, &RIGHT);
2378
2379                 if (cur == NULL)
2380                         rval = XFS_ILOG_DEXT;
2381                 else {
2382                         rval = 0;
2383                         error = xfs_bmbt_lookup_eq(cur, &old, &i);
2384                         if (error)
2385                                 goto done;
2386                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2387                                 xfs_btree_mark_sick(cur);
2388                                 error = -EFSCORRUPTED;
2389                                 goto done;
2390                         }
2391                         error = xfs_bmbt_update(cur, &PREV);
2392                         if (error)
2393                                 goto done;
2394                         error = xfs_btree_increment(cur, 0, &i);
2395                         if (error)
2396                                 goto done;
2397                         error = xfs_bmbt_update(cur, &RIGHT);
2398                         if (error)
2399                                 goto done;
2400                 }
2401                 break;
2402
2403         case BMAP_RIGHT_FILLING:
2404                 /*
2405                  * Setting the last part of a previous oldext extent to newext.
2406                  * The right neighbor is not contiguous.
2407                  */
2408                 old = PREV;
2409                 PREV.br_blockcount -= new->br_blockcount;
2410
2411                 xfs_iext_update_extent(ip, state, icur, &PREV);
2412                 xfs_iext_next(ifp, icur);
2413                 xfs_iext_insert(ip, icur, new, state);
2414                 ifp->if_nextents++;
2415
2416                 if (cur == NULL)
2417                         rval = XFS_ILOG_CORE | XFS_ILOG_DEXT;
2418                 else {
2419                         rval = XFS_ILOG_CORE;
2420                         error = xfs_bmbt_lookup_eq(cur, &old, &i);
2421                         if (error)
2422                                 goto done;
2423                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2424                                 xfs_btree_mark_sick(cur);
2425                                 error = -EFSCORRUPTED;
2426                                 goto done;
2427                         }
2428                         error = xfs_bmbt_update(cur, &PREV);
2429                         if (error)
2430                                 goto done;
2431                         error = xfs_bmbt_lookup_eq(cur, new, &i);
2432                         if (error)
2433                                 goto done;
2434                         if (XFS_IS_CORRUPT(mp, i != 0)) {
2435                                 xfs_btree_mark_sick(cur);
2436                                 error = -EFSCORRUPTED;
2437                                 goto done;
2438                         }
2439                         if ((error = xfs_btree_insert(cur, &i)))
2440                                 goto done;
2441                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2442                                 xfs_btree_mark_sick(cur);
2443                                 error = -EFSCORRUPTED;
2444                                 goto done;
2445                         }
2446                 }
2447                 break;
2448
2449         case 0:
2450                 /*
2451                  * Setting the middle part of a previous oldext extent to
2452                  * newext.  Contiguity is impossible here.
2453                  * One extent becomes three extents.
2454                  */
2455                 old = PREV;
2456                 PREV.br_blockcount = new->br_startoff - PREV.br_startoff;
2457
2458                 r[0] = *new;
2459                 r[1].br_startoff = new_endoff;
2460                 r[1].br_blockcount =
2461                         old.br_startoff + old.br_blockcount - new_endoff;
2462                 r[1].br_startblock = new->br_startblock + new->br_blockcount;
2463                 r[1].br_state = PREV.br_state;
2464
2465                 xfs_iext_update_extent(ip, state, icur, &PREV);
2466                 xfs_iext_next(ifp, icur);
2467                 xfs_iext_insert(ip, icur, &r[1], state);
2468                 xfs_iext_insert(ip, icur, &r[0], state);
2469                 ifp->if_nextents += 2;
2470
2471                 if (cur == NULL)
2472                         rval = XFS_ILOG_CORE | XFS_ILOG_DEXT;
2473                 else {
2474                         rval = XFS_ILOG_CORE;
2475                         error = xfs_bmbt_lookup_eq(cur, &old, &i);
2476                         if (error)
2477                                 goto done;
2478                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2479                                 xfs_btree_mark_sick(cur);
2480                                 error = -EFSCORRUPTED;
2481                                 goto done;
2482                         }
2483                         /* new right extent - oldext */
2484                         error = xfs_bmbt_update(cur, &r[1]);
2485                         if (error)
2486                                 goto done;
2487                         /* new left extent - oldext */
2488                         cur->bc_rec.b = PREV;
2489                         if ((error = xfs_btree_insert(cur, &i)))
2490                                 goto done;
2491                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2492                                 xfs_btree_mark_sick(cur);
2493                                 error = -EFSCORRUPTED;
2494                                 goto done;
2495                         }
2496                         /*
2497                          * Reset the cursor to the position of the new extent
2498                          * we are about to insert as we can't trust it after
2499                          * the previous insert.
2500                          */
2501                         error = xfs_bmbt_lookup_eq(cur, new, &i);
2502                         if (error)
2503                                 goto done;
2504                         if (XFS_IS_CORRUPT(mp, i != 0)) {
2505                                 xfs_btree_mark_sick(cur);
2506                                 error = -EFSCORRUPTED;
2507                                 goto done;
2508                         }
2509                         /* new middle extent - newext */
2510                         if ((error = xfs_btree_insert(cur, &i)))
2511                                 goto done;
2512                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2513                                 xfs_btree_mark_sick(cur);
2514                                 error = -EFSCORRUPTED;
2515                                 goto done;
2516                         }
2517                 }
2518                 break;
2519
2520         case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG:
2521         case BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG:
2522         case BMAP_LEFT_FILLING | BMAP_RIGHT_CONTIG:
2523         case BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG:
2524         case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG:
2525         case BMAP_LEFT_CONTIG:
2526         case BMAP_RIGHT_CONTIG:
2527                 /*
2528                  * These cases are all impossible.
2529                  */
2530                 ASSERT(0);
2531         }
2532
2533         /* update reverse mappings */
2534         xfs_rmap_convert_extent(mp, tp, ip, whichfork, new);
2535
2536         /* convert to a btree if necessary */
2537         if (xfs_bmap_needs_btree(ip, whichfork)) {
2538                 int     tmp_logflags;   /* partial log flag return val */
2539
2540                 ASSERT(cur == NULL);
2541                 error = xfs_bmap_extents_to_btree(tp, ip, &cur, 0,
2542                                 &tmp_logflags, whichfork);
2543                 *logflagsp |= tmp_logflags;
2544                 if (error)
2545                         goto done;
2546         }
2547
2548         /* clear out the allocated field, done with it now in any case. */
2549         if (cur) {
2550                 cur->bc_bmap.allocated = 0;
2551                 *curp = cur;
2552         }
2553
2554         xfs_bmap_check_leaf_extents(*curp, ip, whichfork);
2555 done:
2556         *logflagsp |= rval;
2557         return error;
2558 #undef  LEFT
2559 #undef  RIGHT
2560 #undef  PREV
2561 }
2562
2563 /*
2564  * Convert a hole to a delayed allocation.
2565  */
2566 STATIC void
2567 xfs_bmap_add_extent_hole_delay(
2568         xfs_inode_t             *ip,    /* incore inode pointer */
2569         int                     whichfork,
2570         struct xfs_iext_cursor  *icur,
2571         xfs_bmbt_irec_t         *new)   /* new data to add to file extents */
2572 {
2573         struct xfs_ifork        *ifp;   /* inode fork pointer */
2574         xfs_bmbt_irec_t         left;   /* left neighbor extent entry */
2575         xfs_filblks_t           newlen=0;       /* new indirect size */
2576         xfs_filblks_t           oldlen=0;       /* old indirect size */
2577         xfs_bmbt_irec_t         right;  /* right neighbor extent entry */
2578         uint32_t                state = xfs_bmap_fork_to_state(whichfork);
2579         xfs_filblks_t           temp;    /* temp for indirect calculations */
2580
2581         ifp = xfs_ifork_ptr(ip, whichfork);
2582         ASSERT(isnullstartblock(new->br_startblock));
2583
2584         /*
2585          * Check and set flags if this segment has a left neighbor
2586          */
2587         if (xfs_iext_peek_prev_extent(ifp, icur, &left)) {
2588                 state |= BMAP_LEFT_VALID;
2589                 if (isnullstartblock(left.br_startblock))
2590                         state |= BMAP_LEFT_DELAY;
2591         }
2592
2593         /*
2594          * Check and set flags if the current (right) segment exists.
2595          * If it doesn't exist, we're converting the hole at end-of-file.
2596          */
2597         if (xfs_iext_get_extent(ifp, icur, &right)) {
2598                 state |= BMAP_RIGHT_VALID;
2599                 if (isnullstartblock(right.br_startblock))
2600                         state |= BMAP_RIGHT_DELAY;
2601         }
2602
2603         /*
2604          * Set contiguity flags on the left and right neighbors.
2605          * Don't let extents get too large, even if the pieces are contiguous.
2606          */
2607         if ((state & BMAP_LEFT_VALID) && (state & BMAP_LEFT_DELAY) &&
2608             left.br_startoff + left.br_blockcount == new->br_startoff &&
2609             left.br_blockcount + new->br_blockcount <= XFS_MAX_BMBT_EXTLEN)
2610                 state |= BMAP_LEFT_CONTIG;
2611
2612         if ((state & BMAP_RIGHT_VALID) && (state & BMAP_RIGHT_DELAY) &&
2613             new->br_startoff + new->br_blockcount == right.br_startoff &&
2614             new->br_blockcount + right.br_blockcount <= XFS_MAX_BMBT_EXTLEN &&
2615             (!(state & BMAP_LEFT_CONTIG) ||
2616              (left.br_blockcount + new->br_blockcount +
2617               right.br_blockcount <= XFS_MAX_BMBT_EXTLEN)))
2618                 state |= BMAP_RIGHT_CONTIG;
2619
2620         /*
2621          * Switch out based on the contiguity flags.
2622          */
2623         switch (state & (BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG)) {
2624         case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG:
2625                 /*
2626                  * New allocation is contiguous with delayed allocations
2627                  * on the left and on the right.
2628                  * Merge all three into a single extent record.
2629                  */
2630                 temp = left.br_blockcount + new->br_blockcount +
2631                         right.br_blockcount;
2632
2633                 oldlen = startblockval(left.br_startblock) +
2634                         startblockval(new->br_startblock) +
2635                         startblockval(right.br_startblock);
2636                 newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp),
2637                                          oldlen);
2638                 left.br_startblock = nullstartblock(newlen);
2639                 left.br_blockcount = temp;
2640
2641                 xfs_iext_remove(ip, icur, state);
2642                 xfs_iext_prev(ifp, icur);
2643                 xfs_iext_update_extent(ip, state, icur, &left);
2644                 break;
2645
2646         case BMAP_LEFT_CONTIG:
2647                 /*
2648                  * New allocation is contiguous with a delayed allocation
2649                  * on the left.
2650                  * Merge the new allocation with the left neighbor.
2651                  */
2652                 temp = left.br_blockcount + new->br_blockcount;
2653
2654                 oldlen = startblockval(left.br_startblock) +
2655                         startblockval(new->br_startblock);
2656                 newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp),
2657                                          oldlen);
2658                 left.br_blockcount = temp;
2659                 left.br_startblock = nullstartblock(newlen);
2660
2661                 xfs_iext_prev(ifp, icur);
2662                 xfs_iext_update_extent(ip, state, icur, &left);
2663                 break;
2664
2665         case BMAP_RIGHT_CONTIG:
2666                 /*
2667                  * New allocation is contiguous with a delayed allocation
2668                  * on the right.
2669                  * Merge the new allocation with the right neighbor.
2670                  */
2671                 temp = new->br_blockcount + right.br_blockcount;
2672                 oldlen = startblockval(new->br_startblock) +
2673                         startblockval(right.br_startblock);
2674                 newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp),
2675                                          oldlen);
2676                 right.br_startoff = new->br_startoff;
2677                 right.br_startblock = nullstartblock(newlen);
2678                 right.br_blockcount = temp;
2679                 xfs_iext_update_extent(ip, state, icur, &right);
2680                 break;
2681
2682         case 0:
2683                 /*
2684                  * New allocation is not contiguous with another
2685                  * delayed allocation.
2686                  * Insert a new entry.
2687                  */
2688                 oldlen = newlen = 0;
2689                 xfs_iext_insert(ip, icur, new, state);
2690                 break;
2691         }
2692         if (oldlen != newlen) {
2693                 ASSERT(oldlen > newlen);
2694                 xfs_add_fdblocks(ip->i_mount, oldlen - newlen);
2695
2696                 /*
2697                  * Nothing to do for disk quota accounting here.
2698                  */
2699                 xfs_mod_delalloc(ip, 0, (int64_t)newlen - oldlen);
2700         }
2701 }
2702
2703 /*
2704  * Convert a hole to a real allocation.
2705  */
2706 STATIC int                              /* error */
2707 xfs_bmap_add_extent_hole_real(
2708         struct xfs_trans        *tp,
2709         struct xfs_inode        *ip,
2710         int                     whichfork,
2711         struct xfs_iext_cursor  *icur,
2712         struct xfs_btree_cur    **curp,
2713         struct xfs_bmbt_irec    *new,
2714         int                     *logflagsp,
2715         uint32_t                flags)
2716 {
2717         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
2718         struct xfs_mount        *mp = ip->i_mount;
2719         struct xfs_btree_cur    *cur = *curp;
2720         int                     error;  /* error return value */
2721         int                     i;      /* temp state */
2722         xfs_bmbt_irec_t         left;   /* left neighbor extent entry */
2723         xfs_bmbt_irec_t         right;  /* right neighbor extent entry */
2724         int                     rval=0; /* return value (logging flags) */
2725         uint32_t                state = xfs_bmap_fork_to_state(whichfork);
2726         struct xfs_bmbt_irec    old;
2727
2728         ASSERT(!isnullstartblock(new->br_startblock));
2729         ASSERT(!cur || !(cur->bc_flags & XFS_BTREE_BMBT_WASDEL));
2730
2731         XFS_STATS_INC(mp, xs_add_exlist);
2732
2733         /*
2734          * Check and set flags if this segment has a left neighbor.
2735          */
2736         if (xfs_iext_peek_prev_extent(ifp, icur, &left)) {
2737                 state |= BMAP_LEFT_VALID;
2738                 if (isnullstartblock(left.br_startblock))
2739                         state |= BMAP_LEFT_DELAY;
2740         }
2741
2742         /*
2743          * Check and set flags if this segment has a current value.
2744          * Not true if we're inserting into the "hole" at eof.
2745          */
2746         if (xfs_iext_get_extent(ifp, icur, &right)) {
2747                 state |= BMAP_RIGHT_VALID;
2748                 if (isnullstartblock(right.br_startblock))
2749                         state |= BMAP_RIGHT_DELAY;
2750         }
2751
2752         /*
2753          * We're inserting a real allocation between "left" and "right".
2754          * Set the contiguity flags.  Don't let extents get too large.
2755          */
2756         if ((state & BMAP_LEFT_VALID) && !(state & BMAP_LEFT_DELAY) &&
2757             left.br_startoff + left.br_blockcount == new->br_startoff &&
2758             left.br_startblock + left.br_blockcount == new->br_startblock &&
2759             left.br_state == new->br_state &&
2760             left.br_blockcount + new->br_blockcount <= XFS_MAX_BMBT_EXTLEN)
2761                 state |= BMAP_LEFT_CONTIG;
2762
2763         if ((state & BMAP_RIGHT_VALID) && !(state & BMAP_RIGHT_DELAY) &&
2764             new->br_startoff + new->br_blockcount == right.br_startoff &&
2765             new->br_startblock + new->br_blockcount == right.br_startblock &&
2766             new->br_state == right.br_state &&
2767             new->br_blockcount + right.br_blockcount <= XFS_MAX_BMBT_EXTLEN &&
2768             (!(state & BMAP_LEFT_CONTIG) ||
2769              left.br_blockcount + new->br_blockcount +
2770              right.br_blockcount <= XFS_MAX_BMBT_EXTLEN))
2771                 state |= BMAP_RIGHT_CONTIG;
2772
2773         error = 0;
2774         /*
2775          * Select which case we're in here, and implement it.
2776          */
2777         switch (state & (BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG)) {
2778         case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG:
2779                 /*
2780                  * New allocation is contiguous with real allocations on the
2781                  * left and on the right.
2782                  * Merge all three into a single extent record.
2783                  */
2784                 left.br_blockcount += new->br_blockcount + right.br_blockcount;
2785
2786                 xfs_iext_remove(ip, icur, state);
2787                 xfs_iext_prev(ifp, icur);
2788                 xfs_iext_update_extent(ip, state, icur, &left);
2789                 ifp->if_nextents--;
2790
2791                 if (cur == NULL) {
2792                         rval = XFS_ILOG_CORE | xfs_ilog_fext(whichfork);
2793                 } else {
2794                         rval = XFS_ILOG_CORE;
2795                         error = xfs_bmbt_lookup_eq(cur, &right, &i);
2796                         if (error)
2797                                 goto done;
2798                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2799                                 xfs_btree_mark_sick(cur);
2800                                 error = -EFSCORRUPTED;
2801                                 goto done;
2802                         }
2803                         error = xfs_btree_delete(cur, &i);
2804                         if (error)
2805                                 goto done;
2806                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2807                                 xfs_btree_mark_sick(cur);
2808                                 error = -EFSCORRUPTED;
2809                                 goto done;
2810                         }
2811                         error = xfs_btree_decrement(cur, 0, &i);
2812                         if (error)
2813                                 goto done;
2814                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2815                                 xfs_btree_mark_sick(cur);
2816                                 error = -EFSCORRUPTED;
2817                                 goto done;
2818                         }
2819                         error = xfs_bmbt_update(cur, &left);
2820                         if (error)
2821                                 goto done;
2822                 }
2823                 break;
2824
2825         case BMAP_LEFT_CONTIG:
2826                 /*
2827                  * New allocation is contiguous with a real allocation
2828                  * on the left.
2829                  * Merge the new allocation with the left neighbor.
2830                  */
2831                 old = left;
2832                 left.br_blockcount += new->br_blockcount;
2833
2834                 xfs_iext_prev(ifp, icur);
2835                 xfs_iext_update_extent(ip, state, icur, &left);
2836
2837                 if (cur == NULL) {
2838                         rval = xfs_ilog_fext(whichfork);
2839                 } else {
2840                         rval = 0;
2841                         error = xfs_bmbt_lookup_eq(cur, &old, &i);
2842                         if (error)
2843                                 goto done;
2844                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2845                                 xfs_btree_mark_sick(cur);
2846                                 error = -EFSCORRUPTED;
2847                                 goto done;
2848                         }
2849                         error = xfs_bmbt_update(cur, &left);
2850                         if (error)
2851                                 goto done;
2852                 }
2853                 break;
2854
2855         case BMAP_RIGHT_CONTIG:
2856                 /*
2857                  * New allocation is contiguous with a real allocation
2858                  * on the right.
2859                  * Merge the new allocation with the right neighbor.
2860                  */
2861                 old = right;
2862
2863                 right.br_startoff = new->br_startoff;
2864                 right.br_startblock = new->br_startblock;
2865                 right.br_blockcount += new->br_blockcount;
2866                 xfs_iext_update_extent(ip, state, icur, &right);
2867
2868                 if (cur == NULL) {
2869                         rval = xfs_ilog_fext(whichfork);
2870                 } else {
2871                         rval = 0;
2872                         error = xfs_bmbt_lookup_eq(cur, &old, &i);
2873                         if (error)
2874                                 goto done;
2875                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2876                                 xfs_btree_mark_sick(cur);
2877                                 error = -EFSCORRUPTED;
2878                                 goto done;
2879                         }
2880                         error = xfs_bmbt_update(cur, &right);
2881                         if (error)
2882                                 goto done;
2883                 }
2884                 break;
2885
2886         case 0:
2887                 /*
2888                  * New allocation is not contiguous with another
2889                  * real allocation.
2890                  * Insert a new entry.
2891                  */
2892                 xfs_iext_insert(ip, icur, new, state);
2893                 ifp->if_nextents++;
2894
2895                 if (cur == NULL) {
2896                         rval = XFS_ILOG_CORE | xfs_ilog_fext(whichfork);
2897                 } else {
2898                         rval = XFS_ILOG_CORE;
2899                         error = xfs_bmbt_lookup_eq(cur, new, &i);
2900                         if (error)
2901                                 goto done;
2902                         if (XFS_IS_CORRUPT(mp, i != 0)) {
2903                                 xfs_btree_mark_sick(cur);
2904                                 error = -EFSCORRUPTED;
2905                                 goto done;
2906                         }
2907                         error = xfs_btree_insert(cur, &i);
2908                         if (error)
2909                                 goto done;
2910                         if (XFS_IS_CORRUPT(mp, i != 1)) {
2911                                 xfs_btree_mark_sick(cur);
2912                                 error = -EFSCORRUPTED;
2913                                 goto done;
2914                         }
2915                 }
2916                 break;
2917         }
2918
2919         /* add reverse mapping unless caller opted out */
2920         if (!(flags & XFS_BMAPI_NORMAP))
2921                 xfs_rmap_map_extent(tp, ip, whichfork, new);
2922
2923         /* convert to a btree if necessary */
2924         if (xfs_bmap_needs_btree(ip, whichfork)) {
2925                 int     tmp_logflags;   /* partial log flag return val */
2926
2927                 ASSERT(cur == NULL);
2928                 error = xfs_bmap_extents_to_btree(tp, ip, curp, 0,
2929                                 &tmp_logflags, whichfork);
2930                 *logflagsp |= tmp_logflags;
2931                 cur = *curp;
2932                 if (error)
2933                         goto done;
2934         }
2935
2936         /* clear out the allocated field, done with it now in any case. */
2937         if (cur)
2938                 cur->bc_bmap.allocated = 0;
2939
2940         xfs_bmap_check_leaf_extents(cur, ip, whichfork);
2941 done:
2942         *logflagsp |= rval;
2943         return error;
2944 }
2945
2946 /*
2947  * Functions used in the extent read, allocate and remove paths
2948  */
2949
2950 /*
2951  * Adjust the size of the new extent based on i_extsize and rt extsize.
2952  */
2953 int
2954 xfs_bmap_extsize_align(
2955         xfs_mount_t     *mp,
2956         xfs_bmbt_irec_t *gotp,          /* next extent pointer */
2957         xfs_bmbt_irec_t *prevp,         /* previous extent pointer */
2958         xfs_extlen_t    extsz,          /* align to this extent size */
2959         int             rt,             /* is this a realtime inode? */
2960         int             eof,            /* is extent at end-of-file? */
2961         int             delay,          /* creating delalloc extent? */
2962         int             convert,        /* overwriting unwritten extent? */
2963         xfs_fileoff_t   *offp,          /* in/out: aligned offset */
2964         xfs_extlen_t    *lenp)          /* in/out: aligned length */
2965 {
2966         xfs_fileoff_t   orig_off;       /* original offset */
2967         xfs_extlen_t    orig_alen;      /* original length */
2968         xfs_fileoff_t   orig_end;       /* original off+len */
2969         xfs_fileoff_t   nexto;          /* next file offset */
2970         xfs_fileoff_t   prevo;          /* previous file offset */
2971         xfs_fileoff_t   align_off;      /* temp for offset */
2972         xfs_extlen_t    align_alen;     /* temp for length */
2973         xfs_extlen_t    temp;           /* temp for calculations */
2974
2975         if (convert)
2976                 return 0;
2977
2978         orig_off = align_off = *offp;
2979         orig_alen = align_alen = *lenp;
2980         orig_end = orig_off + orig_alen;
2981
2982         /*
2983          * If this request overlaps an existing extent, then don't
2984          * attempt to perform any additional alignment.
2985          */
2986         if (!delay && !eof &&
2987             (orig_off >= gotp->br_startoff) &&
2988             (orig_end <= gotp->br_startoff + gotp->br_blockcount)) {
2989                 return 0;
2990         }
2991
2992         /*
2993          * If the file offset is unaligned vs. the extent size
2994          * we need to align it.  This will be possible unless
2995          * the file was previously written with a kernel that didn't
2996          * perform this alignment, or if a truncate shot us in the
2997          * foot.
2998          */
2999         div_u64_rem(orig_off, extsz, &temp);
3000         if (temp) {
3001                 align_alen += temp;
3002                 align_off -= temp;
3003         }
3004
3005         /* Same adjustment for the end of the requested area. */
3006         temp = (align_alen % extsz);
3007         if (temp)
3008                 align_alen += extsz - temp;
3009
3010         /*
3011          * For large extent hint sizes, the aligned extent might be larger than
3012          * XFS_BMBT_MAX_EXTLEN. In that case, reduce the size by an extsz so
3013          * that it pulls the length back under XFS_BMBT_MAX_EXTLEN. The outer
3014          * allocation loops handle short allocation just fine, so it is safe to
3015          * do this. We only want to do it when we are forced to, though, because
3016          * it means more allocation operations are required.
3017          */
3018         while (align_alen > XFS_MAX_BMBT_EXTLEN)
3019                 align_alen -= extsz;
3020         ASSERT(align_alen <= XFS_MAX_BMBT_EXTLEN);
3021
3022         /*
3023          * If the previous block overlaps with this proposed allocation
3024          * then move the start forward without adjusting the length.
3025          */
3026         if (prevp->br_startoff != NULLFILEOFF) {
3027                 if (prevp->br_startblock == HOLESTARTBLOCK)
3028                         prevo = prevp->br_startoff;
3029                 else
3030                         prevo = prevp->br_startoff + prevp->br_blockcount;
3031         } else
3032                 prevo = 0;
3033         if (align_off != orig_off && align_off < prevo)
3034                 align_off = prevo;
3035         /*
3036          * If the next block overlaps with this proposed allocation
3037          * then move the start back without adjusting the length,
3038          * but not before offset 0.
3039          * This may of course make the start overlap previous block,
3040          * and if we hit the offset 0 limit then the next block
3041          * can still overlap too.
3042          */
3043         if (!eof && gotp->br_startoff != NULLFILEOFF) {
3044                 if ((delay && gotp->br_startblock == HOLESTARTBLOCK) ||
3045                     (!delay && gotp->br_startblock == DELAYSTARTBLOCK))
3046                         nexto = gotp->br_startoff + gotp->br_blockcount;
3047                 else
3048                         nexto = gotp->br_startoff;
3049         } else
3050                 nexto = NULLFILEOFF;
3051         if (!eof &&
3052             align_off + align_alen != orig_end &&
3053             align_off + align_alen > nexto)
3054                 align_off = nexto > align_alen ? nexto - align_alen : 0;
3055         /*
3056          * If we're now overlapping the next or previous extent that
3057          * means we can't fit an extsz piece in this hole.  Just move
3058          * the start forward to the first valid spot and set
3059          * the length so we hit the end.
3060          */
3061         if (align_off != orig_off && align_off < prevo)
3062                 align_off = prevo;
3063         if (align_off + align_alen != orig_end &&
3064             align_off + align_alen > nexto &&
3065             nexto != NULLFILEOFF) {
3066                 ASSERT(nexto > prevo);
3067                 align_alen = nexto - align_off;
3068         }
3069
3070         /*
3071          * If realtime, and the result isn't a multiple of the realtime
3072          * extent size we need to remove blocks until it is.
3073          */
3074         if (rt && (temp = xfs_extlen_to_rtxmod(mp, align_alen))) {
3075                 /*
3076                  * We're not covering the original request, or
3077                  * we won't be able to once we fix the length.
3078                  */
3079                 if (orig_off < align_off ||
3080                     orig_end > align_off + align_alen ||
3081                     align_alen - temp < orig_alen)
3082                         return -EINVAL;
3083                 /*
3084                  * Try to fix it by moving the start up.
3085                  */
3086                 if (align_off + temp <= orig_off) {
3087                         align_alen -= temp;
3088                         align_off += temp;
3089                 }
3090                 /*
3091                  * Try to fix it by moving the end in.
3092                  */
3093                 else if (align_off + align_alen - temp >= orig_end)
3094                         align_alen -= temp;
3095                 /*
3096                  * Set the start to the minimum then trim the length.
3097                  */
3098                 else {
3099                         align_alen -= orig_off - align_off;
3100                         align_off = orig_off;
3101                         align_alen -= xfs_extlen_to_rtxmod(mp, align_alen);
3102                 }
3103                 /*
3104                  * Result doesn't cover the request, fail it.
3105                  */
3106                 if (orig_off < align_off || orig_end > align_off + align_alen)
3107                         return -EINVAL;
3108         } else {
3109                 ASSERT(orig_off >= align_off);
3110                 /* see XFS_BMBT_MAX_EXTLEN handling above */
3111                 ASSERT(orig_end <= align_off + align_alen ||
3112                        align_alen + extsz > XFS_MAX_BMBT_EXTLEN);
3113         }
3114
3115 #ifdef DEBUG
3116         if (!eof && gotp->br_startoff != NULLFILEOFF)
3117                 ASSERT(align_off + align_alen <= gotp->br_startoff);
3118         if (prevp->br_startoff != NULLFILEOFF)
3119                 ASSERT(align_off >= prevp->br_startoff + prevp->br_blockcount);
3120 #endif
3121
3122         *lenp = align_alen;
3123         *offp = align_off;
3124         return 0;
3125 }
3126
3127 #define XFS_ALLOC_GAP_UNITS     4
3128
3129 /* returns true if ap->blkno was modified */
3130 bool
3131 xfs_bmap_adjacent(
3132         struct xfs_bmalloca     *ap)    /* bmap alloc argument struct */
3133 {
3134         xfs_fsblock_t   adjust;         /* adjustment to block numbers */
3135         xfs_mount_t     *mp;            /* mount point structure */
3136         int             rt;             /* true if inode is realtime */
3137
3138 #define ISVALID(x,y)    \
3139         (rt ? \
3140                 (x) < mp->m_sb.sb_rblocks : \
3141                 XFS_FSB_TO_AGNO(mp, x) == XFS_FSB_TO_AGNO(mp, y) && \
3142                 XFS_FSB_TO_AGNO(mp, x) < mp->m_sb.sb_agcount && \
3143                 XFS_FSB_TO_AGBNO(mp, x) < mp->m_sb.sb_agblocks)
3144
3145         mp = ap->ip->i_mount;
3146         rt = XFS_IS_REALTIME_INODE(ap->ip) &&
3147                 (ap->datatype & XFS_ALLOC_USERDATA);
3148         /*
3149          * If allocating at eof, and there's a previous real block,
3150          * try to use its last block as our starting point.
3151          */
3152         if (ap->eof && ap->prev.br_startoff != NULLFILEOFF &&
3153             !isnullstartblock(ap->prev.br_startblock) &&
3154             ISVALID(ap->prev.br_startblock + ap->prev.br_blockcount,
3155                     ap->prev.br_startblock)) {
3156                 ap->blkno = ap->prev.br_startblock + ap->prev.br_blockcount;
3157                 /*
3158                  * Adjust for the gap between prevp and us.
3159                  */
3160                 adjust = ap->offset -
3161                         (ap->prev.br_startoff + ap->prev.br_blockcount);
3162                 if (adjust &&
3163                     ISVALID(ap->blkno + adjust, ap->prev.br_startblock))
3164                         ap->blkno += adjust;
3165                 return true;
3166         }
3167         /*
3168          * If not at eof, then compare the two neighbor blocks.
3169          * Figure out whether either one gives us a good starting point,
3170          * and pick the better one.
3171          */
3172         if (!ap->eof) {
3173                 xfs_fsblock_t   gotbno;         /* right side block number */
3174                 xfs_fsblock_t   gotdiff=0;      /* right side difference */
3175                 xfs_fsblock_t   prevbno;        /* left side block number */
3176                 xfs_fsblock_t   prevdiff=0;     /* left side difference */
3177
3178                 /*
3179                  * If there's a previous (left) block, select a requested
3180                  * start block based on it.
3181                  */
3182                 if (ap->prev.br_startoff != NULLFILEOFF &&
3183                     !isnullstartblock(ap->prev.br_startblock) &&
3184                     (prevbno = ap->prev.br_startblock +
3185                                ap->prev.br_blockcount) &&
3186                     ISVALID(prevbno, ap->prev.br_startblock)) {
3187                         /*
3188                          * Calculate gap to end of previous block.
3189                          */
3190                         adjust = prevdiff = ap->offset -
3191                                 (ap->prev.br_startoff +
3192                                  ap->prev.br_blockcount);
3193                         /*
3194                          * Figure the startblock based on the previous block's
3195                          * end and the gap size.
3196                          * Heuristic!
3197                          * If the gap is large relative to the piece we're
3198                          * allocating, or using it gives us an invalid block
3199                          * number, then just use the end of the previous block.
3200                          */
3201                         if (prevdiff <= XFS_ALLOC_GAP_UNITS * ap->length &&
3202                             ISVALID(prevbno + prevdiff,
3203                                     ap->prev.br_startblock))
3204                                 prevbno += adjust;
3205                         else
3206                                 prevdiff += adjust;
3207                 }
3208                 /*
3209                  * No previous block or can't follow it, just default.
3210                  */
3211                 else
3212                         prevbno = NULLFSBLOCK;
3213                 /*
3214                  * If there's a following (right) block, select a requested
3215                  * start block based on it.
3216                  */
3217                 if (!isnullstartblock(ap->got.br_startblock)) {
3218                         /*
3219                          * Calculate gap to start of next block.
3220                          */
3221                         adjust = gotdiff = ap->got.br_startoff - ap->offset;
3222                         /*
3223                          * Figure the startblock based on the next block's
3224                          * start and the gap size.
3225                          */
3226                         gotbno = ap->got.br_startblock;
3227                         /*
3228                          * Heuristic!
3229                          * If the gap is large relative to the piece we're
3230                          * allocating, or using it gives us an invalid block
3231                          * number, then just use the start of the next block
3232                          * offset by our length.
3233                          */
3234                         if (gotdiff <= XFS_ALLOC_GAP_UNITS * ap->length &&
3235                             ISVALID(gotbno - gotdiff, gotbno))
3236                                 gotbno -= adjust;
3237                         else if (ISVALID(gotbno - ap->length, gotbno)) {
3238                                 gotbno -= ap->length;
3239                                 gotdiff += adjust - ap->length;
3240                         } else
3241                                 gotdiff += adjust;
3242                 }
3243                 /*
3244                  * No next block, just default.
3245                  */
3246                 else
3247                         gotbno = NULLFSBLOCK;
3248                 /*
3249                  * If both valid, pick the better one, else the only good
3250                  * one, else ap->blkno is already set (to 0 or the inode block).
3251                  */
3252                 if (prevbno != NULLFSBLOCK && gotbno != NULLFSBLOCK) {
3253                         ap->blkno = prevdiff <= gotdiff ? prevbno : gotbno;
3254                         return true;
3255                 }
3256                 if (prevbno != NULLFSBLOCK) {
3257                         ap->blkno = prevbno;
3258                         return true;
3259                 }
3260                 if (gotbno != NULLFSBLOCK) {
3261                         ap->blkno = gotbno;
3262                         return true;
3263                 }
3264         }
3265 #undef ISVALID
3266         return false;
3267 }
3268
3269 int
3270 xfs_bmap_longest_free_extent(
3271         struct xfs_perag        *pag,
3272         struct xfs_trans        *tp,
3273         xfs_extlen_t            *blen)
3274 {
3275         xfs_extlen_t            longest;
3276         int                     error = 0;
3277
3278         if (!xfs_perag_initialised_agf(pag)) {
3279                 error = xfs_alloc_read_agf(pag, tp, XFS_ALLOC_FLAG_TRYLOCK,
3280                                 NULL);
3281                 if (error)
3282                         return error;
3283         }
3284
3285         longest = xfs_alloc_longest_free_extent(pag,
3286                                 xfs_alloc_min_freelist(pag->pag_mount, pag),
3287                                 xfs_ag_resv_needed(pag, XFS_AG_RESV_NONE));
3288         if (*blen < longest)
3289                 *blen = longest;
3290
3291         return 0;
3292 }
3293
3294 static xfs_extlen_t
3295 xfs_bmap_select_minlen(
3296         struct xfs_bmalloca     *ap,
3297         struct xfs_alloc_arg    *args,
3298         xfs_extlen_t            blen)
3299 {
3300
3301         /*
3302          * Since we used XFS_ALLOC_FLAG_TRYLOCK in _longest_free_extent(), it is
3303          * possible that there is enough contiguous free space for this request.
3304          */
3305         if (blen < ap->minlen)
3306                 return ap->minlen;
3307
3308         /*
3309          * If the best seen length is less than the request length,
3310          * use the best as the minimum, otherwise we've got the maxlen we
3311          * were asked for.
3312          */
3313         if (blen < args->maxlen)
3314                 return blen;
3315         return args->maxlen;
3316 }
3317
3318 static int
3319 xfs_bmap_btalloc_select_lengths(
3320         struct xfs_bmalloca     *ap,
3321         struct xfs_alloc_arg    *args,
3322         xfs_extlen_t            *blen)
3323 {
3324         struct xfs_mount        *mp = args->mp;
3325         struct xfs_perag        *pag;
3326         xfs_agnumber_t          agno, startag;
3327         int                     error = 0;
3328
3329         if (ap->tp->t_flags & XFS_TRANS_LOWMODE) {
3330                 args->total = ap->minlen;
3331                 args->minlen = ap->minlen;
3332                 return 0;
3333         }
3334
3335         args->total = ap->total;
3336         startag = XFS_FSB_TO_AGNO(mp, ap->blkno);
3337         if (startag == NULLAGNUMBER)
3338                 startag = 0;
3339
3340         *blen = 0;
3341         for_each_perag_wrap(mp, startag, agno, pag) {
3342                 error = xfs_bmap_longest_free_extent(pag, args->tp, blen);
3343                 if (error && error != -EAGAIN)
3344                         break;
3345                 error = 0;
3346                 if (*blen >= args->maxlen)
3347                         break;
3348         }
3349         if (pag)
3350                 xfs_perag_rele(pag);
3351
3352         args->minlen = xfs_bmap_select_minlen(ap, args, *blen);
3353         return error;
3354 }
3355
3356 /* Update all inode and quota accounting for the allocation we just did. */
3357 void
3358 xfs_bmap_alloc_account(
3359         struct xfs_bmalloca     *ap)
3360 {
3361         bool                    isrt = XFS_IS_REALTIME_INODE(ap->ip) &&
3362                                         !(ap->flags & XFS_BMAPI_ATTRFORK);
3363         uint                    fld;
3364
3365         if (ap->flags & XFS_BMAPI_COWFORK) {
3366                 /*
3367                  * COW fork blocks are in-core only and thus are treated as
3368                  * in-core quota reservation (like delalloc blocks) even when
3369                  * converted to real blocks. The quota reservation is not
3370                  * accounted to disk until blocks are remapped to the data
3371                  * fork. So if these blocks were previously delalloc, we
3372                  * already have quota reservation and there's nothing to do
3373                  * yet.
3374                  */
3375                 if (ap->wasdel) {
3376                         xfs_mod_delalloc(ap->ip, -(int64_t)ap->length, 0);
3377                         return;
3378                 }
3379
3380                 /*
3381                  * Otherwise, we've allocated blocks in a hole. The transaction
3382                  * has acquired in-core quota reservation for this extent.
3383                  * Rather than account these as real blocks, however, we reduce
3384                  * the transaction quota reservation based on the allocation.
3385                  * This essentially transfers the transaction quota reservation
3386                  * to that of a delalloc extent.
3387                  */
3388                 ap->ip->i_delayed_blks += ap->length;
3389                 xfs_trans_mod_dquot_byino(ap->tp, ap->ip, isrt ?
3390                                 XFS_TRANS_DQ_RES_RTBLKS : XFS_TRANS_DQ_RES_BLKS,
3391                                 -(long)ap->length);
3392                 return;
3393         }
3394
3395         /* data/attr fork only */
3396         ap->ip->i_nblocks += ap->length;
3397         xfs_trans_log_inode(ap->tp, ap->ip, XFS_ILOG_CORE);
3398         if (ap->wasdel) {
3399                 ap->ip->i_delayed_blks -= ap->length;
3400                 xfs_mod_delalloc(ap->ip, -(int64_t)ap->length, 0);
3401                 fld = isrt ? XFS_TRANS_DQ_DELRTBCOUNT : XFS_TRANS_DQ_DELBCOUNT;
3402         } else {
3403                 fld = isrt ? XFS_TRANS_DQ_RTBCOUNT : XFS_TRANS_DQ_BCOUNT;
3404         }
3405
3406         xfs_trans_mod_dquot_byino(ap->tp, ap->ip, fld, ap->length);
3407 }
3408
3409 static int
3410 xfs_bmap_compute_alignments(
3411         struct xfs_bmalloca     *ap,
3412         struct xfs_alloc_arg    *args)
3413 {
3414         struct xfs_mount        *mp = args->mp;
3415         xfs_extlen_t            align = 0; /* minimum allocation alignment */
3416         int                     stripe_align = 0;
3417
3418         /* stripe alignment for allocation is determined by mount parameters */
3419         if (mp->m_swidth && xfs_has_swalloc(mp))
3420                 stripe_align = mp->m_swidth;
3421         else if (mp->m_dalign)
3422                 stripe_align = mp->m_dalign;
3423
3424         if (ap->flags & XFS_BMAPI_COWFORK)
3425                 align = xfs_get_cowextsz_hint(ap->ip);
3426         else if (ap->datatype & XFS_ALLOC_USERDATA)
3427                 align = xfs_get_extsz_hint(ap->ip);
3428         if (align) {
3429                 if (xfs_bmap_extsize_align(mp, &ap->got, &ap->prev, align, 0,
3430                                         ap->eof, 0, ap->conv, &ap->offset,
3431                                         &ap->length))
3432                         ASSERT(0);
3433                 ASSERT(ap->length);
3434         }
3435
3436         /* apply extent size hints if obtained earlier */
3437         if (align) {
3438                 args->prod = align;
3439                 div_u64_rem(ap->offset, args->prod, &args->mod);
3440                 if (args->mod)
3441                         args->mod = args->prod - args->mod;
3442         } else if (mp->m_sb.sb_blocksize >= PAGE_SIZE) {
3443                 args->prod = 1;
3444                 args->mod = 0;
3445         } else {
3446                 args->prod = PAGE_SIZE >> mp->m_sb.sb_blocklog;
3447                 div_u64_rem(ap->offset, args->prod, &args->mod);
3448                 if (args->mod)
3449                         args->mod = args->prod - args->mod;
3450         }
3451
3452         return stripe_align;
3453 }
3454
3455 static void
3456 xfs_bmap_process_allocated_extent(
3457         struct xfs_bmalloca     *ap,
3458         struct xfs_alloc_arg    *args,
3459         xfs_fileoff_t           orig_offset,
3460         xfs_extlen_t            orig_length)
3461 {
3462         ap->blkno = args->fsbno;
3463         ap->length = args->len;
3464         /*
3465          * If the extent size hint is active, we tried to round the
3466          * caller's allocation request offset down to extsz and the
3467          * length up to another extsz boundary.  If we found a free
3468          * extent we mapped it in starting at this new offset.  If the
3469          * newly mapped space isn't long enough to cover any of the
3470          * range of offsets that was originally requested, move the
3471          * mapping up so that we can fill as much of the caller's
3472          * original request as possible.  Free space is apparently
3473          * very fragmented so we're unlikely to be able to satisfy the
3474          * hints anyway.
3475          */
3476         if (ap->length <= orig_length)
3477                 ap->offset = orig_offset;
3478         else if (ap->offset + ap->length < orig_offset + orig_length)
3479                 ap->offset = orig_offset + orig_length - ap->length;
3480         xfs_bmap_alloc_account(ap);
3481 }
3482
3483 #ifdef DEBUG
3484 static int
3485 xfs_bmap_exact_minlen_extent_alloc(
3486         struct xfs_bmalloca     *ap)
3487 {
3488         struct xfs_mount        *mp = ap->ip->i_mount;
3489         struct xfs_alloc_arg    args = { .tp = ap->tp, .mp = mp };
3490         xfs_fileoff_t           orig_offset;
3491         xfs_extlen_t            orig_length;
3492         int                     error;
3493
3494         ASSERT(ap->length);
3495
3496         if (ap->minlen != 1) {
3497                 ap->blkno = NULLFSBLOCK;
3498                 ap->length = 0;
3499                 return 0;
3500         }
3501
3502         orig_offset = ap->offset;
3503         orig_length = ap->length;
3504
3505         args.alloc_minlen_only = 1;
3506
3507         xfs_bmap_compute_alignments(ap, &args);
3508
3509         /*
3510          * Unlike the longest extent available in an AG, we don't track
3511          * the length of an AG's shortest extent.
3512          * XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT is a debug only knob and
3513          * hence we can afford to start traversing from the 0th AG since
3514          * we need not be concerned about a drop in performance in
3515          * "debug only" code paths.
3516          */
3517         ap->blkno = XFS_AGB_TO_FSB(mp, 0, 0);
3518
3519         args.oinfo = XFS_RMAP_OINFO_SKIP_UPDATE;
3520         args.minlen = args.maxlen = ap->minlen;
3521         args.total = ap->total;
3522
3523         args.alignment = 1;
3524         args.minalignslop = 0;
3525
3526         args.minleft = ap->minleft;
3527         args.wasdel = ap->wasdel;
3528         args.resv = XFS_AG_RESV_NONE;
3529         args.datatype = ap->datatype;
3530
3531         error = xfs_alloc_vextent_first_ag(&args, ap->blkno);
3532         if (error)
3533                 return error;
3534
3535         if (args.fsbno != NULLFSBLOCK) {
3536                 xfs_bmap_process_allocated_extent(ap, &args, orig_offset,
3537                         orig_length);
3538         } else {
3539                 ap->blkno = NULLFSBLOCK;
3540                 ap->length = 0;
3541         }
3542
3543         return 0;
3544 }
3545 #else
3546
3547 #define xfs_bmap_exact_minlen_extent_alloc(bma) (-EFSCORRUPTED)
3548
3549 #endif
3550
3551 /*
3552  * If we are not low on available data blocks and we are allocating at
3553  * EOF, optimise allocation for contiguous file extension and/or stripe
3554  * alignment of the new extent.
3555  *
3556  * NOTE: ap->aeof is only set if the allocation length is >= the
3557  * stripe unit and the allocation offset is at the end of file.
3558  */
3559 static int
3560 xfs_bmap_btalloc_at_eof(
3561         struct xfs_bmalloca     *ap,
3562         struct xfs_alloc_arg    *args,
3563         xfs_extlen_t            blen,
3564         int                     stripe_align,
3565         bool                    ag_only)
3566 {
3567         struct xfs_mount        *mp = args->mp;
3568         struct xfs_perag        *caller_pag = args->pag;
3569         int                     error;
3570
3571         /*
3572          * If there are already extents in the file, try an exact EOF block
3573          * allocation to extend the file as a contiguous extent. If that fails,
3574          * or it's the first allocation in a file, just try for a stripe aligned
3575          * allocation.
3576          */
3577         if (ap->offset) {
3578                 xfs_extlen_t    nextminlen = 0;
3579
3580                 /*
3581                  * Compute the minlen+alignment for the next case.  Set slop so
3582                  * that the value of minlen+alignment+slop doesn't go up between
3583                  * the calls.
3584                  */
3585                 args->alignment = 1;
3586                 if (blen > stripe_align && blen <= args->maxlen)
3587                         nextminlen = blen - stripe_align;
3588                 else
3589                         nextminlen = args->minlen;
3590                 if (nextminlen + stripe_align > args->minlen + 1)
3591                         args->minalignslop = nextminlen + stripe_align -
3592                                         args->minlen - 1;
3593                 else
3594                         args->minalignslop = 0;
3595
3596                 if (!caller_pag)
3597                         args->pag = xfs_perag_get(mp, XFS_FSB_TO_AGNO(mp, ap->blkno));
3598                 error = xfs_alloc_vextent_exact_bno(args, ap->blkno);
3599                 if (!caller_pag) {
3600                         xfs_perag_put(args->pag);
3601                         args->pag = NULL;
3602                 }
3603                 if (error)
3604                         return error;
3605
3606                 if (args->fsbno != NULLFSBLOCK)
3607                         return 0;
3608                 /*
3609                  * Exact allocation failed. Reset to try an aligned allocation
3610                  * according to the original allocation specification.
3611                  */
3612                 args->alignment = stripe_align;
3613                 args->minlen = nextminlen;
3614                 args->minalignslop = 0;
3615         } else {
3616                 /*
3617                  * Adjust minlen to try and preserve alignment if we
3618                  * can't guarantee an aligned maxlen extent.
3619                  */
3620                 args->alignment = stripe_align;
3621                 if (blen > args->alignment &&
3622                     blen <= args->maxlen + args->alignment)
3623                         args->minlen = blen - args->alignment;
3624                 args->minalignslop = 0;
3625         }
3626
3627         if (ag_only) {
3628                 error = xfs_alloc_vextent_near_bno(args, ap->blkno);
3629         } else {
3630                 args->pag = NULL;
3631                 error = xfs_alloc_vextent_start_ag(args, ap->blkno);
3632                 ASSERT(args->pag == NULL);
3633                 args->pag = caller_pag;
3634         }
3635         if (error)
3636                 return error;
3637
3638         if (args->fsbno != NULLFSBLOCK)
3639                 return 0;
3640
3641         /*
3642          * Allocation failed, so turn return the allocation args to their
3643          * original non-aligned state so the caller can proceed on allocation
3644          * failure as if this function was never called.
3645          */
3646         args->alignment = 1;
3647         return 0;
3648 }
3649
3650 /*
3651  * We have failed multiple allocation attempts so now are in a low space
3652  * allocation situation. Try a locality first full filesystem minimum length
3653  * allocation whilst still maintaining necessary total block reservation
3654  * requirements.
3655  *
3656  * If that fails, we are now critically low on space, so perform a last resort
3657  * allocation attempt: no reserve, no locality, blocking, minimum length, full
3658  * filesystem free space scan. We also indicate to future allocations in this
3659  * transaction that we are critically low on space so they don't waste time on
3660  * allocation modes that are unlikely to succeed.
3661  */
3662 int
3663 xfs_bmap_btalloc_low_space(
3664         struct xfs_bmalloca     *ap,
3665         struct xfs_alloc_arg    *args)
3666 {
3667         int                     error;
3668
3669         if (args->minlen > ap->minlen) {
3670                 args->minlen = ap->minlen;
3671                 error = xfs_alloc_vextent_start_ag(args, ap->blkno);
3672                 if (error || args->fsbno != NULLFSBLOCK)
3673                         return error;
3674         }
3675
3676         /* Last ditch attempt before failure is declared. */
3677         args->total = ap->minlen;
3678         error = xfs_alloc_vextent_first_ag(args, 0);
3679         if (error)
3680                 return error;
3681         ap->tp->t_flags |= XFS_TRANS_LOWMODE;
3682         return 0;
3683 }
3684
3685 static int
3686 xfs_bmap_btalloc_filestreams(
3687         struct xfs_bmalloca     *ap,
3688         struct xfs_alloc_arg    *args,
3689         int                     stripe_align)
3690 {
3691         xfs_extlen_t            blen = 0;
3692         int                     error = 0;
3693
3694
3695         error = xfs_filestream_select_ag(ap, args, &blen);
3696         if (error)
3697                 return error;
3698         ASSERT(args->pag);
3699
3700         /*
3701          * If we are in low space mode, then optimal allocation will fail so
3702          * prepare for minimal allocation and jump to the low space algorithm
3703          * immediately.
3704          */
3705         if (ap->tp->t_flags & XFS_TRANS_LOWMODE) {
3706                 args->minlen = ap->minlen;
3707                 ASSERT(args->fsbno == NULLFSBLOCK);
3708                 goto out_low_space;
3709         }
3710
3711         args->minlen = xfs_bmap_select_minlen(ap, args, blen);
3712         if (ap->aeof)
3713                 error = xfs_bmap_btalloc_at_eof(ap, args, blen, stripe_align,
3714                                 true);
3715
3716         if (!error && args->fsbno == NULLFSBLOCK)
3717                 error = xfs_alloc_vextent_near_bno(args, ap->blkno);
3718
3719 out_low_space:
3720         /*
3721          * We are now done with the perag reference for the filestreams
3722          * association provided by xfs_filestream_select_ag(). Release it now as
3723          * we've either succeeded, had a fatal error or we are out of space and
3724          * need to do a full filesystem scan for free space which will take it's
3725          * own references.
3726          */
3727         xfs_perag_rele(args->pag);
3728         args->pag = NULL;
3729         if (error || args->fsbno != NULLFSBLOCK)
3730                 return error;
3731
3732         return xfs_bmap_btalloc_low_space(ap, args);
3733 }
3734
3735 static int
3736 xfs_bmap_btalloc_best_length(
3737         struct xfs_bmalloca     *ap,
3738         struct xfs_alloc_arg    *args,
3739         int                     stripe_align)
3740 {
3741         xfs_extlen_t            blen = 0;
3742         int                     error;
3743
3744         ap->blkno = XFS_INO_TO_FSB(args->mp, ap->ip->i_ino);
3745         xfs_bmap_adjacent(ap);
3746
3747         /*
3748          * Search for an allocation group with a single extent large enough for
3749          * the request.  If one isn't found, then adjust the minimum allocation
3750          * size to the largest space found.
3751          */
3752         error = xfs_bmap_btalloc_select_lengths(ap, args, &blen);
3753         if (error)
3754                 return error;
3755
3756         /*
3757          * Don't attempt optimal EOF allocation if previous allocations barely
3758          * succeeded due to being near ENOSPC. It is highly unlikely we'll get
3759          * optimal or even aligned allocations in this case, so don't waste time
3760          * trying.
3761          */
3762         if (ap->aeof && !(ap->tp->t_flags & XFS_TRANS_LOWMODE)) {
3763                 error = xfs_bmap_btalloc_at_eof(ap, args, blen, stripe_align,
3764                                 false);
3765                 if (error || args->fsbno != NULLFSBLOCK)
3766                         return error;
3767         }
3768
3769         error = xfs_alloc_vextent_start_ag(args, ap->blkno);
3770         if (error || args->fsbno != NULLFSBLOCK)
3771                 return error;
3772
3773         return xfs_bmap_btalloc_low_space(ap, args);
3774 }
3775
3776 static int
3777 xfs_bmap_btalloc(
3778         struct xfs_bmalloca     *ap)
3779 {
3780         struct xfs_mount        *mp = ap->ip->i_mount;
3781         struct xfs_alloc_arg    args = {
3782                 .tp             = ap->tp,
3783                 .mp             = mp,
3784                 .fsbno          = NULLFSBLOCK,
3785                 .oinfo          = XFS_RMAP_OINFO_SKIP_UPDATE,
3786                 .minleft        = ap->minleft,
3787                 .wasdel         = ap->wasdel,
3788                 .resv           = XFS_AG_RESV_NONE,
3789                 .datatype       = ap->datatype,
3790                 .alignment      = 1,
3791                 .minalignslop   = 0,
3792         };
3793         xfs_fileoff_t           orig_offset;
3794         xfs_extlen_t            orig_length;
3795         int                     error;
3796         int                     stripe_align;
3797
3798         ASSERT(ap->length);
3799         orig_offset = ap->offset;
3800         orig_length = ap->length;
3801
3802         stripe_align = xfs_bmap_compute_alignments(ap, &args);
3803
3804         /* Trim the allocation back to the maximum an AG can fit. */
3805         args.maxlen = min(ap->length, mp->m_ag_max_usable);
3806
3807         if ((ap->datatype & XFS_ALLOC_USERDATA) &&
3808             xfs_inode_is_filestream(ap->ip))
3809                 error = xfs_bmap_btalloc_filestreams(ap, &args, stripe_align);
3810         else
3811                 error = xfs_bmap_btalloc_best_length(ap, &args, stripe_align);
3812         if (error)
3813                 return error;
3814
3815         if (args.fsbno != NULLFSBLOCK) {
3816                 xfs_bmap_process_allocated_extent(ap, &args, orig_offset,
3817                         orig_length);
3818         } else {
3819                 ap->blkno = NULLFSBLOCK;
3820                 ap->length = 0;
3821         }
3822         return 0;
3823 }
3824
3825 /* Trim extent to fit a logical block range. */
3826 void
3827 xfs_trim_extent(
3828         struct xfs_bmbt_irec    *irec,
3829         xfs_fileoff_t           bno,
3830         xfs_filblks_t           len)
3831 {
3832         xfs_fileoff_t           distance;
3833         xfs_fileoff_t           end = bno + len;
3834
3835         if (irec->br_startoff + irec->br_blockcount <= bno ||
3836             irec->br_startoff >= end) {
3837                 irec->br_blockcount = 0;
3838                 return;
3839         }
3840
3841         if (irec->br_startoff < bno) {
3842                 distance = bno - irec->br_startoff;
3843                 if (isnullstartblock(irec->br_startblock))
3844                         irec->br_startblock = DELAYSTARTBLOCK;
3845                 if (irec->br_startblock != DELAYSTARTBLOCK &&
3846                     irec->br_startblock != HOLESTARTBLOCK)
3847                         irec->br_startblock += distance;
3848                 irec->br_startoff += distance;
3849                 irec->br_blockcount -= distance;
3850         }
3851
3852         if (end < irec->br_startoff + irec->br_blockcount) {
3853                 distance = irec->br_startoff + irec->br_blockcount - end;
3854                 irec->br_blockcount -= distance;
3855         }
3856 }
3857
3858 /*
3859  * Trim the returned map to the required bounds
3860  */
3861 STATIC void
3862 xfs_bmapi_trim_map(
3863         struct xfs_bmbt_irec    *mval,
3864         struct xfs_bmbt_irec    *got,
3865         xfs_fileoff_t           *bno,
3866         xfs_filblks_t           len,
3867         xfs_fileoff_t           obno,
3868         xfs_fileoff_t           end,
3869         int                     n,
3870         uint32_t                flags)
3871 {
3872         if ((flags & XFS_BMAPI_ENTIRE) ||
3873             got->br_startoff + got->br_blockcount <= obno) {
3874                 *mval = *got;
3875                 if (isnullstartblock(got->br_startblock))
3876                         mval->br_startblock = DELAYSTARTBLOCK;
3877                 return;
3878         }
3879
3880         if (obno > *bno)
3881                 *bno = obno;
3882         ASSERT((*bno >= obno) || (n == 0));
3883         ASSERT(*bno < end);
3884         mval->br_startoff = *bno;
3885         if (isnullstartblock(got->br_startblock))
3886                 mval->br_startblock = DELAYSTARTBLOCK;
3887         else
3888                 mval->br_startblock = got->br_startblock +
3889                                         (*bno - got->br_startoff);
3890         /*
3891          * Return the minimum of what we got and what we asked for for
3892          * the length.  We can use the len variable here because it is
3893          * modified below and we could have been there before coming
3894          * here if the first part of the allocation didn't overlap what
3895          * was asked for.
3896          */
3897         mval->br_blockcount = XFS_FILBLKS_MIN(end - *bno,
3898                         got->br_blockcount - (*bno - got->br_startoff));
3899         mval->br_state = got->br_state;
3900         ASSERT(mval->br_blockcount <= len);
3901         return;
3902 }
3903
3904 /*
3905  * Update and validate the extent map to return
3906  */
3907 STATIC void
3908 xfs_bmapi_update_map(
3909         struct xfs_bmbt_irec    **map,
3910         xfs_fileoff_t           *bno,
3911         xfs_filblks_t           *len,
3912         xfs_fileoff_t           obno,
3913         xfs_fileoff_t           end,
3914         int                     *n,
3915         uint32_t                flags)
3916 {
3917         xfs_bmbt_irec_t *mval = *map;
3918
3919         ASSERT((flags & XFS_BMAPI_ENTIRE) ||
3920                ((mval->br_startoff + mval->br_blockcount) <= end));
3921         ASSERT((flags & XFS_BMAPI_ENTIRE) || (mval->br_blockcount <= *len) ||
3922                (mval->br_startoff < obno));
3923
3924         *bno = mval->br_startoff + mval->br_blockcount;
3925         *len = end - *bno;
3926         if (*n > 0 && mval->br_startoff == mval[-1].br_startoff) {
3927                 /* update previous map with new information */
3928                 ASSERT(mval->br_startblock == mval[-1].br_startblock);
3929                 ASSERT(mval->br_blockcount > mval[-1].br_blockcount);
3930                 ASSERT(mval->br_state == mval[-1].br_state);
3931                 mval[-1].br_blockcount = mval->br_blockcount;
3932                 mval[-1].br_state = mval->br_state;
3933         } else if (*n > 0 && mval->br_startblock != DELAYSTARTBLOCK &&
3934                    mval[-1].br_startblock != DELAYSTARTBLOCK &&
3935                    mval[-1].br_startblock != HOLESTARTBLOCK &&
3936                    mval->br_startblock == mval[-1].br_startblock +
3937                                           mval[-1].br_blockcount &&
3938                    mval[-1].br_state == mval->br_state) {
3939                 ASSERT(mval->br_startoff ==
3940                        mval[-1].br_startoff + mval[-1].br_blockcount);
3941                 mval[-1].br_blockcount += mval->br_blockcount;
3942         } else if (*n > 0 &&
3943                    mval->br_startblock == DELAYSTARTBLOCK &&
3944                    mval[-1].br_startblock == DELAYSTARTBLOCK &&
3945                    mval->br_startoff ==
3946                    mval[-1].br_startoff + mval[-1].br_blockcount) {
3947                 mval[-1].br_blockcount += mval->br_blockcount;
3948                 mval[-1].br_state = mval->br_state;
3949         } else if (!((*n == 0) &&
3950                      ((mval->br_startoff + mval->br_blockcount) <=
3951                       obno))) {
3952                 mval++;
3953                 (*n)++;
3954         }
3955         *map = mval;
3956 }
3957
3958 /*
3959  * Map file blocks to filesystem blocks without allocation.
3960  */
3961 int
3962 xfs_bmapi_read(
3963         struct xfs_inode        *ip,
3964         xfs_fileoff_t           bno,
3965         xfs_filblks_t           len,
3966         struct xfs_bmbt_irec    *mval,
3967         int                     *nmap,
3968         uint32_t                flags)
3969 {
3970         struct xfs_mount        *mp = ip->i_mount;
3971         int                     whichfork = xfs_bmapi_whichfork(flags);
3972         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
3973         struct xfs_bmbt_irec    got;
3974         xfs_fileoff_t           obno;
3975         xfs_fileoff_t           end;
3976         struct xfs_iext_cursor  icur;
3977         int                     error;
3978         bool                    eof = false;
3979         int                     n = 0;
3980
3981         ASSERT(*nmap >= 1);
3982         ASSERT(!(flags & ~(XFS_BMAPI_ATTRFORK | XFS_BMAPI_ENTIRE)));
3983         xfs_assert_ilocked(ip, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL);
3984
3985         if (WARN_ON_ONCE(!ifp)) {
3986                 xfs_bmap_mark_sick(ip, whichfork);
3987                 return -EFSCORRUPTED;
3988         }
3989
3990         if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) ||
3991             XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) {
3992                 xfs_bmap_mark_sick(ip, whichfork);
3993                 return -EFSCORRUPTED;
3994         }
3995
3996         if (xfs_is_shutdown(mp))
3997                 return -EIO;
3998
3999         XFS_STATS_INC(mp, xs_blk_mapr);
4000
4001         error = xfs_iread_extents(NULL, ip, whichfork);
4002         if (error)
4003                 return error;
4004
4005         if (!xfs_iext_lookup_extent(ip, ifp, bno, &icur, &got))
4006                 eof = true;
4007         end = bno + len;
4008         obno = bno;
4009
4010         while (bno < end && n < *nmap) {
4011                 /* Reading past eof, act as though there's a hole up to end. */
4012                 if (eof)
4013                         got.br_startoff = end;
4014                 if (got.br_startoff > bno) {
4015                         /* Reading in a hole.  */
4016                         mval->br_startoff = bno;
4017                         mval->br_startblock = HOLESTARTBLOCK;
4018                         mval->br_blockcount =
4019                                 XFS_FILBLKS_MIN(len, got.br_startoff - bno);
4020                         mval->br_state = XFS_EXT_NORM;
4021                         bno += mval->br_blockcount;
4022                         len -= mval->br_blockcount;
4023                         mval++;
4024                         n++;
4025                         continue;
4026                 }
4027
4028                 /* set up the extent map to return. */
4029                 xfs_bmapi_trim_map(mval, &got, &bno, len, obno, end, n, flags);
4030                 xfs_bmapi_update_map(&mval, &bno, &len, obno, end, &n, flags);
4031
4032                 /* If we're done, stop now. */
4033                 if (bno >= end || n >= *nmap)
4034                         break;
4035
4036                 /* Else go on to the next record. */
4037                 if (!xfs_iext_next_extent(ifp, &icur, &got))
4038                         eof = true;
4039         }
4040         *nmap = n;
4041         return 0;
4042 }
4043
4044 /*
4045  * Add a delayed allocation extent to an inode. Blocks are reserved from the
4046  * global pool and the extent inserted into the inode in-core extent tree.
4047  *
4048  * On entry, got refers to the first extent beyond the offset of the extent to
4049  * allocate or eof is specified if no such extent exists. On return, got refers
4050  * to the extent record that was inserted to the inode fork.
4051  *
4052  * Note that the allocated extent may have been merged with contiguous extents
4053  * during insertion into the inode fork. Thus, got does not reflect the current
4054  * state of the inode fork on return. If necessary, the caller can use lastx to
4055  * look up the updated record in the inode fork.
4056  */
4057 int
4058 xfs_bmapi_reserve_delalloc(
4059         struct xfs_inode        *ip,
4060         int                     whichfork,
4061         xfs_fileoff_t           off,
4062         xfs_filblks_t           len,
4063         xfs_filblks_t           prealloc,
4064         struct xfs_bmbt_irec    *got,
4065         struct xfs_iext_cursor  *icur,
4066         int                     eof)
4067 {
4068         struct xfs_mount        *mp = ip->i_mount;
4069         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
4070         xfs_extlen_t            alen;
4071         xfs_extlen_t            indlen;
4072         uint64_t                fdblocks;
4073         int                     error;
4074         xfs_fileoff_t           aoff = off;
4075
4076         /*
4077          * Cap the alloc length. Keep track of prealloc so we know whether to
4078          * tag the inode before we return.
4079          */
4080         alen = XFS_FILBLKS_MIN(len + prealloc, XFS_MAX_BMBT_EXTLEN);
4081         if (!eof)
4082                 alen = XFS_FILBLKS_MIN(alen, got->br_startoff - aoff);
4083         if (prealloc && alen >= len)
4084                 prealloc = alen - len;
4085
4086         /* Figure out the extent size, adjust alen */
4087         if (whichfork == XFS_COW_FORK) {
4088                 struct xfs_bmbt_irec    prev;
4089                 xfs_extlen_t            extsz = xfs_get_cowextsz_hint(ip);
4090
4091                 if (!xfs_iext_peek_prev_extent(ifp, icur, &prev))
4092                         prev.br_startoff = NULLFILEOFF;
4093
4094                 error = xfs_bmap_extsize_align(mp, got, &prev, extsz, 0, eof,
4095                                                1, 0, &aoff, &alen);
4096                 ASSERT(!error);
4097         }
4098
4099         /*
4100          * Make a transaction-less quota reservation for delayed allocation
4101          * blocks.  This number gets adjusted later.  We return if we haven't
4102          * allocated blocks already inside this loop.
4103          */
4104         error = xfs_quota_reserve_blkres(ip, alen);
4105         if (error)
4106                 return error;
4107
4108         /*
4109          * Split changing sb for alen and indlen since they could be coming
4110          * from different places.
4111          */
4112         indlen = (xfs_extlen_t)xfs_bmap_worst_indlen(ip, alen);
4113         ASSERT(indlen > 0);
4114
4115         fdblocks = indlen;
4116         if (XFS_IS_REALTIME_INODE(ip)) {
4117                 error = xfs_dec_frextents(mp, xfs_rtb_to_rtx(mp, alen));
4118                 if (error)
4119                         goto out_unreserve_quota;
4120         } else {
4121                 fdblocks += alen;
4122         }
4123
4124         error = xfs_dec_fdblocks(mp, fdblocks, false);
4125         if (error)
4126                 goto out_unreserve_frextents;
4127
4128         ip->i_delayed_blks += alen;
4129         xfs_mod_delalloc(ip, alen, indlen);
4130
4131         got->br_startoff = aoff;
4132         got->br_startblock = nullstartblock(indlen);
4133         got->br_blockcount = alen;
4134         got->br_state = XFS_EXT_NORM;
4135
4136         xfs_bmap_add_extent_hole_delay(ip, whichfork, icur, got);
4137
4138         /*
4139          * Tag the inode if blocks were preallocated. Note that COW fork
4140          * preallocation can occur at the start or end of the extent, even when
4141          * prealloc == 0, so we must also check the aligned offset and length.
4142          */
4143         if (whichfork == XFS_DATA_FORK && prealloc)
4144                 xfs_inode_set_eofblocks_tag(ip);
4145         if (whichfork == XFS_COW_FORK && (prealloc || aoff < off || alen > len))
4146                 xfs_inode_set_cowblocks_tag(ip);
4147
4148         return 0;
4149
4150 out_unreserve_frextents:
4151         if (XFS_IS_REALTIME_INODE(ip))
4152                 xfs_add_frextents(mp, xfs_rtb_to_rtx(mp, alen));
4153 out_unreserve_quota:
4154         if (XFS_IS_QUOTA_ON(mp))
4155                 xfs_quota_unreserve_blkres(ip, alen);
4156         return error;
4157 }
4158
4159 static int
4160 xfs_bmap_alloc_userdata(
4161         struct xfs_bmalloca     *bma)
4162 {
4163         struct xfs_mount        *mp = bma->ip->i_mount;
4164         int                     whichfork = xfs_bmapi_whichfork(bma->flags);
4165         int                     error;
4166
4167         /*
4168          * Set the data type being allocated. For the data fork, the first data
4169          * in the file is treated differently to all other allocations. For the
4170          * attribute fork, we only need to ensure the allocated range is not on
4171          * the busy list.
4172          */
4173         bma->datatype = XFS_ALLOC_NOBUSY;
4174         if (whichfork == XFS_DATA_FORK || whichfork == XFS_COW_FORK) {
4175                 bma->datatype |= XFS_ALLOC_USERDATA;
4176                 if (bma->offset == 0)
4177                         bma->datatype |= XFS_ALLOC_INITIAL_USER_DATA;
4178
4179                 if (mp->m_dalign && bma->length >= mp->m_dalign) {
4180                         error = xfs_bmap_isaeof(bma, whichfork);
4181                         if (error)
4182                                 return error;
4183                 }
4184
4185                 if (XFS_IS_REALTIME_INODE(bma->ip))
4186                         return xfs_bmap_rtalloc(bma);
4187         }
4188
4189         if (unlikely(XFS_TEST_ERROR(false, mp,
4190                         XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT)))
4191                 return xfs_bmap_exact_minlen_extent_alloc(bma);
4192
4193         return xfs_bmap_btalloc(bma);
4194 }
4195
4196 static int
4197 xfs_bmapi_allocate(
4198         struct xfs_bmalloca     *bma)
4199 {
4200         struct xfs_mount        *mp = bma->ip->i_mount;
4201         int                     whichfork = xfs_bmapi_whichfork(bma->flags);
4202         struct xfs_ifork        *ifp = xfs_ifork_ptr(bma->ip, whichfork);
4203         int                     tmp_logflags = 0;
4204         int                     error;
4205
4206         ASSERT(bma->length > 0);
4207
4208         /*
4209          * For the wasdelay case, we could also just allocate the stuff asked
4210          * for in this bmap call but that wouldn't be as good.
4211          */
4212         if (bma->wasdel) {
4213                 bma->length = (xfs_extlen_t)bma->got.br_blockcount;
4214                 bma->offset = bma->got.br_startoff;
4215                 if (!xfs_iext_peek_prev_extent(ifp, &bma->icur, &bma->prev))
4216                         bma->prev.br_startoff = NULLFILEOFF;
4217         } else {
4218                 bma->length = XFS_FILBLKS_MIN(bma->length, XFS_MAX_BMBT_EXTLEN);
4219                 if (!bma->eof)
4220                         bma->length = XFS_FILBLKS_MIN(bma->length,
4221                                         bma->got.br_startoff - bma->offset);
4222         }
4223
4224         if (bma->flags & XFS_BMAPI_CONTIG)
4225                 bma->minlen = bma->length;
4226         else
4227                 bma->minlen = 1;
4228
4229         if (bma->flags & XFS_BMAPI_METADATA) {
4230                 if (unlikely(XFS_TEST_ERROR(false, mp,
4231                                 XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT)))
4232                         error = xfs_bmap_exact_minlen_extent_alloc(bma);
4233                 else
4234                         error = xfs_bmap_btalloc(bma);
4235         } else {
4236                 error = xfs_bmap_alloc_userdata(bma);
4237         }
4238         if (error || bma->blkno == NULLFSBLOCK)
4239                 return error;
4240
4241         if (bma->flags & XFS_BMAPI_ZERO) {
4242                 error = xfs_zero_extent(bma->ip, bma->blkno, bma->length);
4243                 if (error)
4244                         return error;
4245         }
4246
4247         if (ifp->if_format == XFS_DINODE_FMT_BTREE && !bma->cur)
4248                 bma->cur = xfs_bmbt_init_cursor(mp, bma->tp, bma->ip, whichfork);
4249         /*
4250          * Bump the number of extents we've allocated
4251          * in this call.
4252          */
4253         bma->nallocs++;
4254
4255         if (bma->cur && bma->wasdel)
4256                 bma->cur->bc_flags |= XFS_BTREE_BMBT_WASDEL;
4257
4258         bma->got.br_startoff = bma->offset;
4259         bma->got.br_startblock = bma->blkno;
4260         bma->got.br_blockcount = bma->length;
4261         bma->got.br_state = XFS_EXT_NORM;
4262
4263         if (bma->flags & XFS_BMAPI_PREALLOC)
4264                 bma->got.br_state = XFS_EXT_UNWRITTEN;
4265
4266         if (bma->wasdel)
4267                 error = xfs_bmap_add_extent_delay_real(bma, whichfork);
4268         else
4269                 error = xfs_bmap_add_extent_hole_real(bma->tp, bma->ip,
4270                                 whichfork, &bma->icur, &bma->cur, &bma->got,
4271                                 &bma->logflags, bma->flags);
4272
4273         bma->logflags |= tmp_logflags;
4274         if (error)
4275                 return error;
4276
4277         /*
4278          * Update our extent pointer, given that xfs_bmap_add_extent_delay_real
4279          * or xfs_bmap_add_extent_hole_real might have merged it into one of
4280          * the neighbouring ones.
4281          */
4282         xfs_iext_get_extent(ifp, &bma->icur, &bma->got);
4283
4284         ASSERT(bma->got.br_startoff <= bma->offset);
4285         ASSERT(bma->got.br_startoff + bma->got.br_blockcount >=
4286                bma->offset + bma->length);
4287         ASSERT(bma->got.br_state == XFS_EXT_NORM ||
4288                bma->got.br_state == XFS_EXT_UNWRITTEN);
4289         return 0;
4290 }
4291
4292 STATIC int
4293 xfs_bmapi_convert_unwritten(
4294         struct xfs_bmalloca     *bma,
4295         struct xfs_bmbt_irec    *mval,
4296         xfs_filblks_t           len,
4297         uint32_t                flags)
4298 {
4299         int                     whichfork = xfs_bmapi_whichfork(flags);
4300         struct xfs_ifork        *ifp = xfs_ifork_ptr(bma->ip, whichfork);
4301         int                     tmp_logflags = 0;
4302         int                     error;
4303
4304         /* check if we need to do unwritten->real conversion */
4305         if (mval->br_state == XFS_EXT_UNWRITTEN &&
4306             (flags & XFS_BMAPI_PREALLOC))
4307                 return 0;
4308
4309         /* check if we need to do real->unwritten conversion */
4310         if (mval->br_state == XFS_EXT_NORM &&
4311             (flags & (XFS_BMAPI_PREALLOC | XFS_BMAPI_CONVERT)) !=
4312                         (XFS_BMAPI_PREALLOC | XFS_BMAPI_CONVERT))
4313                 return 0;
4314
4315         /*
4316          * Modify (by adding) the state flag, if writing.
4317          */
4318         ASSERT(mval->br_blockcount <= len);
4319         if (ifp->if_format == XFS_DINODE_FMT_BTREE && !bma->cur) {
4320                 bma->cur = xfs_bmbt_init_cursor(bma->ip->i_mount, bma->tp,
4321                                         bma->ip, whichfork);
4322         }
4323         mval->br_state = (mval->br_state == XFS_EXT_UNWRITTEN)
4324                                 ? XFS_EXT_NORM : XFS_EXT_UNWRITTEN;
4325
4326         /*
4327          * Before insertion into the bmbt, zero the range being converted
4328          * if required.
4329          */
4330         if (flags & XFS_BMAPI_ZERO) {
4331                 error = xfs_zero_extent(bma->ip, mval->br_startblock,
4332                                         mval->br_blockcount);
4333                 if (error)
4334                         return error;
4335         }
4336
4337         error = xfs_bmap_add_extent_unwritten_real(bma->tp, bma->ip, whichfork,
4338                         &bma->icur, &bma->cur, mval, &tmp_logflags);
4339         /*
4340          * Log the inode core unconditionally in the unwritten extent conversion
4341          * path because the conversion might not have done so (e.g., if the
4342          * extent count hasn't changed). We need to make sure the inode is dirty
4343          * in the transaction for the sake of fsync(), even if nothing has
4344          * changed, because fsync() will not force the log for this transaction
4345          * unless it sees the inode pinned.
4346          *
4347          * Note: If we're only converting cow fork extents, there aren't
4348          * any on-disk updates to make, so we don't need to log anything.
4349          */
4350         if (whichfork != XFS_COW_FORK)
4351                 bma->logflags |= tmp_logflags | XFS_ILOG_CORE;
4352         if (error)
4353                 return error;
4354
4355         /*
4356          * Update our extent pointer, given that
4357          * xfs_bmap_add_extent_unwritten_real might have merged it into one
4358          * of the neighbouring ones.
4359          */
4360         xfs_iext_get_extent(ifp, &bma->icur, &bma->got);
4361
4362         /*
4363          * We may have combined previously unwritten space with written space,
4364          * so generate another request.
4365          */
4366         if (mval->br_blockcount < len)
4367                 return -EAGAIN;
4368         return 0;
4369 }
4370
4371 xfs_extlen_t
4372 xfs_bmapi_minleft(
4373         struct xfs_trans        *tp,
4374         struct xfs_inode        *ip,
4375         int                     fork)
4376 {
4377         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, fork);
4378
4379         if (tp && tp->t_highest_agno != NULLAGNUMBER)
4380                 return 0;
4381         if (ifp->if_format != XFS_DINODE_FMT_BTREE)
4382                 return 1;
4383         return be16_to_cpu(ifp->if_broot->bb_level) + 1;
4384 }
4385
4386 /*
4387  * Log whatever the flags say, even if error.  Otherwise we might miss detecting
4388  * a case where the data is changed, there's an error, and it's not logged so we
4389  * don't shutdown when we should.  Don't bother logging extents/btree changes if
4390  * we converted to the other format.
4391  */
4392 static void
4393 xfs_bmapi_finish(
4394         struct xfs_bmalloca     *bma,
4395         int                     whichfork,
4396         int                     error)
4397 {
4398         struct xfs_ifork        *ifp = xfs_ifork_ptr(bma->ip, whichfork);
4399
4400         if ((bma->logflags & xfs_ilog_fext(whichfork)) &&
4401             ifp->if_format != XFS_DINODE_FMT_EXTENTS)
4402                 bma->logflags &= ~xfs_ilog_fext(whichfork);
4403         else if ((bma->logflags & xfs_ilog_fbroot(whichfork)) &&
4404                  ifp->if_format != XFS_DINODE_FMT_BTREE)
4405                 bma->logflags &= ~xfs_ilog_fbroot(whichfork);
4406
4407         if (bma->logflags)
4408                 xfs_trans_log_inode(bma->tp, bma->ip, bma->logflags);
4409         if (bma->cur)
4410                 xfs_btree_del_cursor(bma->cur, error);
4411 }
4412
4413 /*
4414  * Map file blocks to filesystem blocks, and allocate blocks or convert the
4415  * extent state if necessary.  Details behaviour is controlled by the flags
4416  * parameter.  Only allocates blocks from a single allocation group, to avoid
4417  * locking problems.
4418  */
4419 int
4420 xfs_bmapi_write(
4421         struct xfs_trans        *tp,            /* transaction pointer */
4422         struct xfs_inode        *ip,            /* incore inode */
4423         xfs_fileoff_t           bno,            /* starting file offs. mapped */
4424         xfs_filblks_t           len,            /* length to map in file */
4425         uint32_t                flags,          /* XFS_BMAPI_... */
4426         xfs_extlen_t            total,          /* total blocks needed */
4427         struct xfs_bmbt_irec    *mval,          /* output: map values */
4428         int                     *nmap)          /* i/o: mval size/count */
4429 {
4430         struct xfs_bmalloca     bma = {
4431                 .tp             = tp,
4432                 .ip             = ip,
4433                 .total          = total,
4434         };
4435         struct xfs_mount        *mp = ip->i_mount;
4436         int                     whichfork = xfs_bmapi_whichfork(flags);
4437         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
4438         xfs_fileoff_t           end;            /* end of mapped file region */
4439         bool                    eof = false;    /* after the end of extents */
4440         int                     error;          /* error return */
4441         int                     n;              /* current extent index */
4442         xfs_fileoff_t           obno;           /* old block number (offset) */
4443
4444 #ifdef DEBUG
4445         xfs_fileoff_t           orig_bno;       /* original block number value */
4446         int                     orig_flags;     /* original flags arg value */
4447         xfs_filblks_t           orig_len;       /* original value of len arg */
4448         struct xfs_bmbt_irec    *orig_mval;     /* original value of mval */
4449         int                     orig_nmap;      /* original value of *nmap */
4450
4451         orig_bno = bno;
4452         orig_len = len;
4453         orig_flags = flags;
4454         orig_mval = mval;
4455         orig_nmap = *nmap;
4456 #endif
4457
4458         ASSERT(*nmap >= 1);
4459         ASSERT(*nmap <= XFS_BMAP_MAX_NMAP);
4460         ASSERT(tp != NULL);
4461         ASSERT(len > 0);
4462         ASSERT(ifp->if_format != XFS_DINODE_FMT_LOCAL);
4463         xfs_assert_ilocked(ip, XFS_ILOCK_EXCL);
4464         ASSERT(!(flags & XFS_BMAPI_REMAP));
4465
4466         /* zeroing is for currently only for data extents, not metadata */
4467         ASSERT((flags & (XFS_BMAPI_METADATA | XFS_BMAPI_ZERO)) !=
4468                         (XFS_BMAPI_METADATA | XFS_BMAPI_ZERO));
4469         /*
4470          * we can allocate unwritten extents or pre-zero allocated blocks,
4471          * but it makes no sense to do both at once. This would result in
4472          * zeroing the unwritten extent twice, but it still being an
4473          * unwritten extent....
4474          */
4475         ASSERT((flags & (XFS_BMAPI_PREALLOC | XFS_BMAPI_ZERO)) !=
4476                         (XFS_BMAPI_PREALLOC | XFS_BMAPI_ZERO));
4477
4478         if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) ||
4479             XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) {
4480                 xfs_bmap_mark_sick(ip, whichfork);
4481                 return -EFSCORRUPTED;
4482         }
4483
4484         if (xfs_is_shutdown(mp))
4485                 return -EIO;
4486
4487         XFS_STATS_INC(mp, xs_blk_mapw);
4488
4489         error = xfs_iread_extents(tp, ip, whichfork);
4490         if (error)
4491                 goto error0;
4492
4493         if (!xfs_iext_lookup_extent(ip, ifp, bno, &bma.icur, &bma.got))
4494                 eof = true;
4495         if (!xfs_iext_peek_prev_extent(ifp, &bma.icur, &bma.prev))
4496                 bma.prev.br_startoff = NULLFILEOFF;
4497         bma.minleft = xfs_bmapi_minleft(tp, ip, whichfork);
4498
4499         n = 0;
4500         end = bno + len;
4501         obno = bno;
4502         while (bno < end && n < *nmap) {
4503                 bool                    need_alloc = false, wasdelay = false;
4504
4505                 /* in hole or beyond EOF? */
4506                 if (eof || bma.got.br_startoff > bno) {
4507                         /*
4508                          * CoW fork conversions should /never/ hit EOF or
4509                          * holes.  There should always be something for us
4510                          * to work on.
4511                          */
4512                         ASSERT(!((flags & XFS_BMAPI_CONVERT) &&
4513                                  (flags & XFS_BMAPI_COWFORK)));
4514
4515                         need_alloc = true;
4516                 } else if (isnullstartblock(bma.got.br_startblock)) {
4517                         wasdelay = true;
4518                 }
4519
4520                 /*
4521                  * First, deal with the hole before the allocated space
4522                  * that we found, if any.
4523                  */
4524                 if (need_alloc || wasdelay) {
4525                         bma.eof = eof;
4526                         bma.conv = !!(flags & XFS_BMAPI_CONVERT);
4527                         bma.wasdel = wasdelay;
4528                         bma.offset = bno;
4529                         bma.flags = flags;
4530
4531                         /*
4532                          * There's a 32/64 bit type mismatch between the
4533                          * allocation length request (which can be 64 bits in
4534                          * length) and the bma length request, which is
4535                          * xfs_extlen_t and therefore 32 bits. Hence we have to
4536                          * check for 32-bit overflows and handle them here.
4537                          */
4538                         if (len > (xfs_filblks_t)XFS_MAX_BMBT_EXTLEN)
4539                                 bma.length = XFS_MAX_BMBT_EXTLEN;
4540                         else
4541                                 bma.length = len;
4542
4543                         ASSERT(len > 0);
4544                         ASSERT(bma.length > 0);
4545                         error = xfs_bmapi_allocate(&bma);
4546                         if (error)
4547                                 goto error0;
4548                         if (bma.blkno == NULLFSBLOCK)
4549                                 break;
4550
4551                         /*
4552                          * If this is a CoW allocation, record the data in
4553                          * the refcount btree for orphan recovery.
4554                          */
4555                         if (whichfork == XFS_COW_FORK)
4556                                 xfs_refcount_alloc_cow_extent(tp, bma.blkno,
4557                                                 bma.length);
4558                 }
4559
4560                 /* Deal with the allocated space we found.  */
4561                 xfs_bmapi_trim_map(mval, &bma.got, &bno, len, obno,
4562                                                         end, n, flags);
4563
4564                 /* Execute unwritten extent conversion if necessary */
4565                 error = xfs_bmapi_convert_unwritten(&bma, mval, len, flags);
4566                 if (error == -EAGAIN)
4567                         continue;
4568                 if (error)
4569                         goto error0;
4570
4571                 /* update the extent map to return */
4572                 xfs_bmapi_update_map(&mval, &bno, &len, obno, end, &n, flags);
4573
4574                 /*
4575                  * If we're done, stop now.  Stop when we've allocated
4576                  * XFS_BMAP_MAX_NMAP extents no matter what.  Otherwise
4577                  * the transaction may get too big.
4578                  */
4579                 if (bno >= end || n >= *nmap || bma.nallocs >= *nmap)
4580                         break;
4581
4582                 /* Else go on to the next record. */
4583                 bma.prev = bma.got;
4584                 if (!xfs_iext_next_extent(ifp, &bma.icur, &bma.got))
4585                         eof = true;
4586         }
4587         *nmap = n;
4588
4589         error = xfs_bmap_btree_to_extents(tp, ip, bma.cur, &bma.logflags,
4590                         whichfork);
4591         if (error)
4592                 goto error0;
4593
4594         ASSERT(ifp->if_format != XFS_DINODE_FMT_BTREE ||
4595                ifp->if_nextents > XFS_IFORK_MAXEXT(ip, whichfork));
4596         xfs_bmapi_finish(&bma, whichfork, 0);
4597         xfs_bmap_validate_ret(orig_bno, orig_len, orig_flags, orig_mval,
4598                 orig_nmap, *nmap);
4599         return 0;
4600 error0:
4601         xfs_bmapi_finish(&bma, whichfork, error);
4602         return error;
4603 }
4604
4605 /*
4606  * Convert an existing delalloc extent to real blocks based on file offset. This
4607  * attempts to allocate the entire delalloc extent and may require multiple
4608  * invocations to allocate the target offset if a large enough physical extent
4609  * is not available.
4610  */
4611 int
4612 xfs_bmapi_convert_delalloc(
4613         struct xfs_inode        *ip,
4614         int                     whichfork,
4615         xfs_off_t               offset,
4616         struct iomap            *iomap,
4617         unsigned int            *seq)
4618 {
4619         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
4620         struct xfs_mount        *mp = ip->i_mount;
4621         xfs_fileoff_t           offset_fsb = XFS_B_TO_FSBT(mp, offset);
4622         struct xfs_bmalloca     bma = { NULL };
4623         uint16_t                flags = 0;
4624         struct xfs_trans        *tp;
4625         int                     error;
4626
4627         if (whichfork == XFS_COW_FORK)
4628                 flags |= IOMAP_F_SHARED;
4629
4630         /*
4631          * Space for the extent and indirect blocks was reserved when the
4632          * delalloc extent was created so there's no need to do so here.
4633          */
4634         error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0,
4635                                 XFS_TRANS_RESERVE, &tp);
4636         if (error)
4637                 return error;
4638
4639         xfs_ilock(ip, XFS_ILOCK_EXCL);
4640         xfs_trans_ijoin(tp, ip, 0);
4641
4642         error = xfs_iext_count_may_overflow(ip, whichfork,
4643                         XFS_IEXT_ADD_NOSPLIT_CNT);
4644         if (error == -EFBIG)
4645                 error = xfs_iext_count_upgrade(tp, ip,
4646                                 XFS_IEXT_ADD_NOSPLIT_CNT);
4647         if (error)
4648                 goto out_trans_cancel;
4649
4650         if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &bma.icur, &bma.got) ||
4651             bma.got.br_startoff > offset_fsb) {
4652                 /*
4653                  * No extent found in the range we are trying to convert.  This
4654                  * should only happen for the COW fork, where another thread
4655                  * might have moved the extent to the data fork in the meantime.
4656                  */
4657                 WARN_ON_ONCE(whichfork != XFS_COW_FORK);
4658                 error = -EAGAIN;
4659                 goto out_trans_cancel;
4660         }
4661
4662         /*
4663          * If we find a real extent here we raced with another thread converting
4664          * the extent.  Just return the real extent at this offset.
4665          */
4666         if (!isnullstartblock(bma.got.br_startblock)) {
4667                 xfs_bmbt_to_iomap(ip, iomap, &bma.got, 0, flags,
4668                                 xfs_iomap_inode_sequence(ip, flags));
4669                 *seq = READ_ONCE(ifp->if_seq);
4670                 goto out_trans_cancel;
4671         }
4672
4673         bma.tp = tp;
4674         bma.ip = ip;
4675         bma.wasdel = true;
4676         bma.offset = bma.got.br_startoff;
4677         bma.length = max_t(xfs_filblks_t, bma.got.br_blockcount,
4678                         XFS_MAX_BMBT_EXTLEN);
4679         bma.minleft = xfs_bmapi_minleft(tp, ip, whichfork);
4680
4681         /*
4682          * When we're converting the delalloc reservations backing dirty pages
4683          * in the page cache, we must be careful about how we create the new
4684          * extents:
4685          *
4686          * New CoW fork extents are created unwritten, turned into real extents
4687          * when we're about to write the data to disk, and mapped into the data
4688          * fork after the write finishes.  End of story.
4689          *
4690          * New data fork extents must be mapped in as unwritten and converted
4691          * to real extents after the write succeeds to avoid exposing stale
4692          * disk contents if we crash.
4693          */
4694         bma.flags = XFS_BMAPI_PREALLOC;
4695         if (whichfork == XFS_COW_FORK)
4696                 bma.flags |= XFS_BMAPI_COWFORK;
4697
4698         if (!xfs_iext_peek_prev_extent(ifp, &bma.icur, &bma.prev))
4699                 bma.prev.br_startoff = NULLFILEOFF;
4700
4701         error = xfs_bmapi_allocate(&bma);
4702         if (error)
4703                 goto out_finish;
4704
4705         error = -ENOSPC;
4706         if (WARN_ON_ONCE(bma.blkno == NULLFSBLOCK))
4707                 goto out_finish;
4708         if (WARN_ON_ONCE(!xfs_valid_startblock(ip, bma.got.br_startblock))) {
4709                 xfs_bmap_mark_sick(ip, whichfork);
4710                 error = -EFSCORRUPTED;
4711                 goto out_finish;
4712         }
4713
4714         XFS_STATS_ADD(mp, xs_xstrat_bytes, XFS_FSB_TO_B(mp, bma.length));
4715         XFS_STATS_INC(mp, xs_xstrat_quick);
4716
4717         ASSERT(!isnullstartblock(bma.got.br_startblock));
4718         xfs_bmbt_to_iomap(ip, iomap, &bma.got, 0, flags,
4719                                 xfs_iomap_inode_sequence(ip, flags));
4720         *seq = READ_ONCE(ifp->if_seq);
4721
4722         if (whichfork == XFS_COW_FORK)
4723                 xfs_refcount_alloc_cow_extent(tp, bma.blkno, bma.length);
4724
4725         error = xfs_bmap_btree_to_extents(tp, ip, bma.cur, &bma.logflags,
4726                         whichfork);
4727         if (error)
4728                 goto out_finish;
4729
4730         xfs_bmapi_finish(&bma, whichfork, 0);
4731         error = xfs_trans_commit(tp);
4732         xfs_iunlock(ip, XFS_ILOCK_EXCL);
4733         return error;
4734
4735 out_finish:
4736         xfs_bmapi_finish(&bma, whichfork, error);
4737 out_trans_cancel:
4738         xfs_trans_cancel(tp);
4739         xfs_iunlock(ip, XFS_ILOCK_EXCL);
4740         return error;
4741 }
4742
4743 int
4744 xfs_bmapi_remap(
4745         struct xfs_trans        *tp,
4746         struct xfs_inode        *ip,
4747         xfs_fileoff_t           bno,
4748         xfs_filblks_t           len,
4749         xfs_fsblock_t           startblock,
4750         uint32_t                flags)
4751 {
4752         struct xfs_mount        *mp = ip->i_mount;
4753         struct xfs_ifork        *ifp;
4754         struct xfs_btree_cur    *cur = NULL;
4755         struct xfs_bmbt_irec    got;
4756         struct xfs_iext_cursor  icur;
4757         int                     whichfork = xfs_bmapi_whichfork(flags);
4758         int                     logflags = 0, error;
4759
4760         ifp = xfs_ifork_ptr(ip, whichfork);
4761         ASSERT(len > 0);
4762         ASSERT(len <= (xfs_filblks_t)XFS_MAX_BMBT_EXTLEN);
4763         xfs_assert_ilocked(ip, XFS_ILOCK_EXCL);
4764         ASSERT(!(flags & ~(XFS_BMAPI_ATTRFORK | XFS_BMAPI_PREALLOC |
4765                            XFS_BMAPI_NORMAP)));
4766         ASSERT((flags & (XFS_BMAPI_ATTRFORK | XFS_BMAPI_PREALLOC)) !=
4767                         (XFS_BMAPI_ATTRFORK | XFS_BMAPI_PREALLOC));
4768
4769         if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) ||
4770             XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) {
4771                 xfs_bmap_mark_sick(ip, whichfork);
4772                 return -EFSCORRUPTED;
4773         }
4774
4775         if (xfs_is_shutdown(mp))
4776                 return -EIO;
4777
4778         error = xfs_iread_extents(tp, ip, whichfork);
4779         if (error)
4780                 return error;
4781
4782         if (xfs_iext_lookup_extent(ip, ifp, bno, &icur, &got)) {
4783                 /* make sure we only reflink into a hole. */
4784                 ASSERT(got.br_startoff > bno);
4785                 ASSERT(got.br_startoff - bno >= len);
4786         }
4787
4788         ip->i_nblocks += len;
4789         xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
4790
4791         if (ifp->if_format == XFS_DINODE_FMT_BTREE)
4792                 cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork);
4793
4794         got.br_startoff = bno;
4795         got.br_startblock = startblock;
4796         got.br_blockcount = len;
4797         if (flags & XFS_BMAPI_PREALLOC)
4798                 got.br_state = XFS_EXT_UNWRITTEN;
4799         else
4800                 got.br_state = XFS_EXT_NORM;
4801
4802         error = xfs_bmap_add_extent_hole_real(tp, ip, whichfork, &icur,
4803                         &cur, &got, &logflags, flags);
4804         if (error)
4805                 goto error0;
4806
4807         error = xfs_bmap_btree_to_extents(tp, ip, cur, &logflags, whichfork);
4808
4809 error0:
4810         if (ip->i_df.if_format != XFS_DINODE_FMT_EXTENTS)
4811                 logflags &= ~XFS_ILOG_DEXT;
4812         else if (ip->i_df.if_format != XFS_DINODE_FMT_BTREE)
4813                 logflags &= ~XFS_ILOG_DBROOT;
4814
4815         if (logflags)
4816                 xfs_trans_log_inode(tp, ip, logflags);
4817         if (cur)
4818                 xfs_btree_del_cursor(cur, error);
4819         return error;
4820 }
4821
4822 /*
4823  * When a delalloc extent is split (e.g., due to a hole punch), the original
4824  * indlen reservation must be shared across the two new extents that are left
4825  * behind.
4826  *
4827  * Given the original reservation and the worst case indlen for the two new
4828  * extents (as calculated by xfs_bmap_worst_indlen()), split the original
4829  * reservation fairly across the two new extents. If necessary, steal available
4830  * blocks from a deleted extent to make up a reservation deficiency (e.g., if
4831  * ores == 1). The number of stolen blocks is returned. The availability and
4832  * subsequent accounting of stolen blocks is the responsibility of the caller.
4833  */
4834 static void
4835 xfs_bmap_split_indlen(
4836         xfs_filblks_t                   ores,           /* original res. */
4837         xfs_filblks_t                   *indlen1,       /* ext1 worst indlen */
4838         xfs_filblks_t                   *indlen2)       /* ext2 worst indlen */
4839 {
4840         xfs_filblks_t                   len1 = *indlen1;
4841         xfs_filblks_t                   len2 = *indlen2;
4842         xfs_filblks_t                   nres = len1 + len2; /* new total res. */
4843         xfs_filblks_t                   resfactor;
4844
4845         /*
4846          * We can't meet the total required reservation for the two extents.
4847          * Calculate the percent of the overall shortage between both extents
4848          * and apply this percentage to each of the requested indlen values.
4849          * This distributes the shortage fairly and reduces the chances that one
4850          * of the two extents is left with nothing when extents are repeatedly
4851          * split.
4852          */
4853         resfactor = (ores * 100);
4854         do_div(resfactor, nres);
4855         len1 *= resfactor;
4856         do_div(len1, 100);
4857         len2 *= resfactor;
4858         do_div(len2, 100);
4859         ASSERT(len1 + len2 <= ores);
4860         ASSERT(len1 < *indlen1 && len2 < *indlen2);
4861
4862         /*
4863          * Hand out the remainder to each extent. If one of the two reservations
4864          * is zero, we want to make sure that one gets a block first. The loop
4865          * below starts with len1, so hand len2 a block right off the bat if it
4866          * is zero.
4867          */
4868         ores -= (len1 + len2);
4869         ASSERT((*indlen1 - len1) + (*indlen2 - len2) >= ores);
4870         if (ores && !len2 && *indlen2) {
4871                 len2++;
4872                 ores--;
4873         }
4874         while (ores) {
4875                 if (len1 < *indlen1) {
4876                         len1++;
4877                         ores--;
4878                 }
4879                 if (!ores)
4880                         break;
4881                 if (len2 < *indlen2) {
4882                         len2++;
4883                         ores--;
4884                 }
4885         }
4886
4887         *indlen1 = len1;
4888         *indlen2 = len2;
4889 }
4890
4891 int
4892 xfs_bmap_del_extent_delay(
4893         struct xfs_inode        *ip,
4894         int                     whichfork,
4895         struct xfs_iext_cursor  *icur,
4896         struct xfs_bmbt_irec    *got,
4897         struct xfs_bmbt_irec    *del)
4898 {
4899         struct xfs_mount        *mp = ip->i_mount;
4900         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
4901         struct xfs_bmbt_irec    new;
4902         int64_t                 da_old, da_new, da_diff = 0;
4903         xfs_fileoff_t           del_endoff, got_endoff;
4904         xfs_filblks_t           got_indlen, new_indlen, stolen = 0;
4905         uint32_t                state = xfs_bmap_fork_to_state(whichfork);
4906         uint64_t                fdblocks;
4907         int                     error = 0;
4908         bool                    isrt;
4909
4910         XFS_STATS_INC(mp, xs_del_exlist);
4911
4912         isrt = xfs_ifork_is_realtime(ip, whichfork);
4913         del_endoff = del->br_startoff + del->br_blockcount;
4914         got_endoff = got->br_startoff + got->br_blockcount;
4915         da_old = startblockval(got->br_startblock);
4916         da_new = 0;
4917
4918         ASSERT(del->br_blockcount > 0);
4919         ASSERT(got->br_startoff <= del->br_startoff);
4920         ASSERT(got_endoff >= del_endoff);
4921
4922         /*
4923          * Update the inode delalloc counter now and wait to update the
4924          * sb counters as we might have to borrow some blocks for the
4925          * indirect block accounting.
4926          */
4927         error = xfs_quota_unreserve_blkres(ip, del->br_blockcount);
4928         if (error)
4929                 return error;
4930         ip->i_delayed_blks -= del->br_blockcount;
4931
4932         if (got->br_startoff == del->br_startoff)
4933                 state |= BMAP_LEFT_FILLING;
4934         if (got_endoff == del_endoff)
4935                 state |= BMAP_RIGHT_FILLING;
4936
4937         switch (state & (BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING)) {
4938         case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING:
4939                 /*
4940                  * Matches the whole extent.  Delete the entry.
4941                  */
4942                 xfs_iext_remove(ip, icur, state);
4943                 xfs_iext_prev(ifp, icur);
4944                 break;
4945         case BMAP_LEFT_FILLING:
4946                 /*
4947                  * Deleting the first part of the extent.
4948                  */
4949                 got->br_startoff = del_endoff;
4950                 got->br_blockcount -= del->br_blockcount;
4951                 da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip,
4952                                 got->br_blockcount), da_old);
4953                 got->br_startblock = nullstartblock((int)da_new);
4954                 xfs_iext_update_extent(ip, state, icur, got);
4955                 break;
4956         case BMAP_RIGHT_FILLING:
4957                 /*
4958                  * Deleting the last part of the extent.
4959                  */
4960                 got->br_blockcount = got->br_blockcount - del->br_blockcount;
4961                 da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip,
4962                                 got->br_blockcount), da_old);
4963                 got->br_startblock = nullstartblock((int)da_new);
4964                 xfs_iext_update_extent(ip, state, icur, got);
4965                 break;
4966         case 0:
4967                 /*
4968                  * Deleting the middle of the extent.
4969                  *
4970                  * Distribute the original indlen reservation across the two new
4971                  * extents.  Steal blocks from the deleted extent if necessary.
4972                  * Stealing blocks simply fudges the fdblocks accounting below.
4973                  * Warn if either of the new indlen reservations is zero as this
4974                  * can lead to delalloc problems.
4975                  */
4976                 got->br_blockcount = del->br_startoff - got->br_startoff;
4977                 got_indlen = xfs_bmap_worst_indlen(ip, got->br_blockcount);
4978
4979                 new.br_blockcount = got_endoff - del_endoff;
4980                 new_indlen = xfs_bmap_worst_indlen(ip, new.br_blockcount);
4981
4982                 WARN_ON_ONCE(!got_indlen || !new_indlen);
4983                 /*
4984                  * Steal as many blocks as we can to try and satisfy the worst
4985                  * case indlen for both new extents.
4986                  *
4987                  * However, we can't just steal reservations from the data
4988                  * blocks if this is an RT inodes as the data and metadata
4989                  * blocks come from different pools.  We'll have to live with
4990                  * under-filled indirect reservation in this case.
4991                  */
4992                 da_new = got_indlen + new_indlen;
4993                 if (da_new > da_old && !isrt) {
4994                         stolen = XFS_FILBLKS_MIN(da_new - da_old,
4995                                                  del->br_blockcount);
4996                         da_old += stolen;
4997                 }
4998                 if (da_new > da_old)
4999                         xfs_bmap_split_indlen(da_old, &got_indlen, &new_indlen);
5000                 da_new = got_indlen + new_indlen;
5001
5002                 got->br_startblock = nullstartblock((int)got_indlen);
5003
5004                 new.br_startoff = del_endoff;
5005                 new.br_state = got->br_state;
5006                 new.br_startblock = nullstartblock((int)new_indlen);
5007
5008                 xfs_iext_update_extent(ip, state, icur, got);
5009                 xfs_iext_next(ifp, icur);
5010                 xfs_iext_insert(ip, icur, &new, state);
5011
5012                 del->br_blockcount -= stolen;
5013                 break;
5014         }
5015
5016         ASSERT(da_old >= da_new);
5017         da_diff = da_old - da_new;
5018         fdblocks = da_diff;
5019
5020         if (isrt)
5021                 xfs_add_frextents(mp, xfs_rtb_to_rtx(mp, del->br_blockcount));
5022         else
5023                 fdblocks += del->br_blockcount;
5024
5025         xfs_add_fdblocks(mp, fdblocks);
5026         xfs_mod_delalloc(ip, -(int64_t)del->br_blockcount, -da_diff);
5027         return error;
5028 }
5029
5030 void
5031 xfs_bmap_del_extent_cow(
5032         struct xfs_inode        *ip,
5033         struct xfs_iext_cursor  *icur,
5034         struct xfs_bmbt_irec    *got,
5035         struct xfs_bmbt_irec    *del)
5036 {
5037         struct xfs_mount        *mp = ip->i_mount;
5038         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, XFS_COW_FORK);
5039         struct xfs_bmbt_irec    new;
5040         xfs_fileoff_t           del_endoff, got_endoff;
5041         uint32_t                state = BMAP_COWFORK;
5042
5043         XFS_STATS_INC(mp, xs_del_exlist);
5044
5045         del_endoff = del->br_startoff + del->br_blockcount;
5046         got_endoff = got->br_startoff + got->br_blockcount;
5047
5048         ASSERT(del->br_blockcount > 0);
5049         ASSERT(got->br_startoff <= del->br_startoff);
5050         ASSERT(got_endoff >= del_endoff);
5051         ASSERT(!isnullstartblock(got->br_startblock));
5052
5053         if (got->br_startoff == del->br_startoff)
5054                 state |= BMAP_LEFT_FILLING;
5055         if (got_endoff == del_endoff)
5056                 state |= BMAP_RIGHT_FILLING;
5057
5058         switch (state & (BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING)) {
5059         case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING:
5060                 /*
5061                  * Matches the whole extent.  Delete the entry.
5062                  */
5063                 xfs_iext_remove(ip, icur, state);
5064                 xfs_iext_prev(ifp, icur);
5065                 break;
5066         case BMAP_LEFT_FILLING:
5067                 /*
5068                  * Deleting the first part of the extent.
5069                  */
5070                 got->br_startoff = del_endoff;
5071                 got->br_blockcount -= del->br_blockcount;
5072                 got->br_startblock = del->br_startblock + del->br_blockcount;
5073                 xfs_iext_update_extent(ip, state, icur, got);
5074                 break;
5075         case BMAP_RIGHT_FILLING:
5076                 /*
5077                  * Deleting the last part of the extent.
5078                  */
5079                 got->br_blockcount -= del->br_blockcount;
5080                 xfs_iext_update_extent(ip, state, icur, got);
5081                 break;
5082         case 0:
5083                 /*
5084                  * Deleting the middle of the extent.
5085                  */
5086                 got->br_blockcount = del->br_startoff - got->br_startoff;
5087
5088                 new.br_startoff = del_endoff;
5089                 new.br_blockcount = got_endoff - del_endoff;
5090                 new.br_state = got->br_state;
5091                 new.br_startblock = del->br_startblock + del->br_blockcount;
5092
5093                 xfs_iext_update_extent(ip, state, icur, got);
5094                 xfs_iext_next(ifp, icur);
5095                 xfs_iext_insert(ip, icur, &new, state);
5096                 break;
5097         }
5098         ip->i_delayed_blks -= del->br_blockcount;
5099 }
5100
5101 /*
5102  * Called by xfs_bmapi to update file extent records and the btree
5103  * after removing space.
5104  */
5105 STATIC int                              /* error */
5106 xfs_bmap_del_extent_real(
5107         xfs_inode_t             *ip,    /* incore inode pointer */
5108         xfs_trans_t             *tp,    /* current transaction pointer */
5109         struct xfs_iext_cursor  *icur,
5110         struct xfs_btree_cur    *cur,   /* if null, not a btree */
5111         xfs_bmbt_irec_t         *del,   /* data to remove from extents */
5112         int                     *logflagsp, /* inode logging flags */
5113         int                     whichfork, /* data or attr fork */
5114         uint32_t                bflags) /* bmapi flags */
5115 {
5116         xfs_fsblock_t           del_endblock=0; /* first block past del */
5117         xfs_fileoff_t           del_endoff;     /* first offset past del */
5118         int                     error = 0;      /* error return value */
5119         struct xfs_bmbt_irec    got;    /* current extent entry */
5120         xfs_fileoff_t           got_endoff;     /* first offset past got */
5121         int                     i;      /* temp state */
5122         struct xfs_ifork        *ifp;   /* inode fork pointer */
5123         xfs_mount_t             *mp;    /* mount structure */
5124         xfs_filblks_t           nblks;  /* quota/sb block count */
5125         xfs_bmbt_irec_t         new;    /* new record to be inserted */
5126         /* REFERENCED */
5127         uint                    qfield; /* quota field to update */
5128         uint32_t                state = xfs_bmap_fork_to_state(whichfork);
5129         struct xfs_bmbt_irec    old;
5130
5131         *logflagsp = 0;
5132
5133         mp = ip->i_mount;
5134         XFS_STATS_INC(mp, xs_del_exlist);
5135
5136         ifp = xfs_ifork_ptr(ip, whichfork);
5137         ASSERT(del->br_blockcount > 0);
5138         xfs_iext_get_extent(ifp, icur, &got);
5139         ASSERT(got.br_startoff <= del->br_startoff);
5140         del_endoff = del->br_startoff + del->br_blockcount;
5141         got_endoff = got.br_startoff + got.br_blockcount;
5142         ASSERT(got_endoff >= del_endoff);
5143         ASSERT(!isnullstartblock(got.br_startblock));
5144         qfield = 0;
5145
5146         /*
5147          * If it's the case where the directory code is running with no block
5148          * reservation, and the deleted block is in the middle of its extent,
5149          * and the resulting insert of an extent would cause transformation to
5150          * btree format, then reject it.  The calling code will then swap blocks
5151          * around instead.  We have to do this now, rather than waiting for the
5152          * conversion to btree format, since the transaction will be dirty then.
5153          */
5154         if (tp->t_blk_res == 0 &&
5155             ifp->if_format == XFS_DINODE_FMT_EXTENTS &&
5156             ifp->if_nextents >= XFS_IFORK_MAXEXT(ip, whichfork) &&
5157             del->br_startoff > got.br_startoff && del_endoff < got_endoff)
5158                 return -ENOSPC;
5159
5160         *logflagsp = XFS_ILOG_CORE;
5161         if (xfs_ifork_is_realtime(ip, whichfork))
5162                 qfield = XFS_TRANS_DQ_RTBCOUNT;
5163         else
5164                 qfield = XFS_TRANS_DQ_BCOUNT;
5165         nblks = del->br_blockcount;
5166
5167         del_endblock = del->br_startblock + del->br_blockcount;
5168         if (cur) {
5169                 error = xfs_bmbt_lookup_eq(cur, &got, &i);
5170                 if (error)
5171                         return error;
5172                 if (XFS_IS_CORRUPT(mp, i != 1)) {
5173                         xfs_btree_mark_sick(cur);
5174                         return -EFSCORRUPTED;
5175                 }
5176         }
5177
5178         if (got.br_startoff == del->br_startoff)
5179                 state |= BMAP_LEFT_FILLING;
5180         if (got_endoff == del_endoff)
5181                 state |= BMAP_RIGHT_FILLING;
5182
5183         switch (state & (BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING)) {
5184         case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING:
5185                 /*
5186                  * Matches the whole extent.  Delete the entry.
5187                  */
5188                 xfs_iext_remove(ip, icur, state);
5189                 xfs_iext_prev(ifp, icur);
5190                 ifp->if_nextents--;
5191
5192                 *logflagsp |= XFS_ILOG_CORE;
5193                 if (!cur) {
5194                         *logflagsp |= xfs_ilog_fext(whichfork);
5195                         break;
5196                 }
5197                 if ((error = xfs_btree_delete(cur, &i)))
5198                         return error;
5199                 if (XFS_IS_CORRUPT(mp, i != 1)) {
5200                         xfs_btree_mark_sick(cur);
5201                         return -EFSCORRUPTED;
5202                 }
5203                 break;
5204         case BMAP_LEFT_FILLING:
5205                 /*
5206                  * Deleting the first part of the extent.
5207                  */
5208                 got.br_startoff = del_endoff;
5209                 got.br_startblock = del_endblock;
5210                 got.br_blockcount -= del->br_blockcount;
5211                 xfs_iext_update_extent(ip, state, icur, &got);
5212                 if (!cur) {
5213                         *logflagsp |= xfs_ilog_fext(whichfork);
5214                         break;
5215                 }
5216                 error = xfs_bmbt_update(cur, &got);
5217                 if (error)
5218                         return error;
5219                 break;
5220         case BMAP_RIGHT_FILLING:
5221                 /*
5222                  * Deleting the last part of the extent.
5223                  */
5224                 got.br_blockcount -= del->br_blockcount;
5225                 xfs_iext_update_extent(ip, state, icur, &got);
5226                 if (!cur) {
5227                         *logflagsp |= xfs_ilog_fext(whichfork);
5228                         break;
5229                 }
5230                 error = xfs_bmbt_update(cur, &got);
5231                 if (error)
5232                         return error;
5233                 break;
5234         case 0:
5235                 /*
5236                  * Deleting the middle of the extent.
5237                  */
5238
5239                 old = got;
5240
5241                 got.br_blockcount = del->br_startoff - got.br_startoff;
5242                 xfs_iext_update_extent(ip, state, icur, &got);
5243
5244                 new.br_startoff = del_endoff;
5245                 new.br_blockcount = got_endoff - del_endoff;
5246                 new.br_state = got.br_state;
5247                 new.br_startblock = del_endblock;
5248
5249                 *logflagsp |= XFS_ILOG_CORE;
5250                 if (cur) {
5251                         error = xfs_bmbt_update(cur, &got);
5252                         if (error)
5253                                 return error;
5254                         error = xfs_btree_increment(cur, 0, &i);
5255                         if (error)
5256                                 return error;
5257                         cur->bc_rec.b = new;
5258                         error = xfs_btree_insert(cur, &i);
5259                         if (error && error != -ENOSPC)
5260                                 return error;
5261                         /*
5262                          * If get no-space back from btree insert, it tried a
5263                          * split, and we have a zero block reservation.  Fix up
5264                          * our state and return the error.
5265                          */
5266                         if (error == -ENOSPC) {
5267                                 /*
5268                                  * Reset the cursor, don't trust it after any
5269                                  * insert operation.
5270                                  */
5271                                 error = xfs_bmbt_lookup_eq(cur, &got, &i);
5272                                 if (error)
5273                                         return error;
5274                                 if (XFS_IS_CORRUPT(mp, i != 1)) {
5275                                         xfs_btree_mark_sick(cur);
5276                                         return -EFSCORRUPTED;
5277                                 }
5278                                 /*
5279                                  * Update the btree record back
5280                                  * to the original value.
5281                                  */
5282                                 error = xfs_bmbt_update(cur, &old);
5283                                 if (error)
5284                                         return error;
5285                                 /*
5286                                  * Reset the extent record back
5287                                  * to the original value.
5288                                  */
5289                                 xfs_iext_update_extent(ip, state, icur, &old);
5290                                 *logflagsp = 0;
5291                                 return -ENOSPC;
5292                         }
5293                         if (XFS_IS_CORRUPT(mp, i != 1)) {
5294                                 xfs_btree_mark_sick(cur);
5295                                 return -EFSCORRUPTED;
5296                         }
5297                 } else
5298                         *logflagsp |= xfs_ilog_fext(whichfork);
5299
5300                 ifp->if_nextents++;
5301                 xfs_iext_next(ifp, icur);
5302                 xfs_iext_insert(ip, icur, &new, state);
5303                 break;
5304         }
5305
5306         /* remove reverse mapping */
5307         xfs_rmap_unmap_extent(tp, ip, whichfork, del);
5308
5309         /*
5310          * If we need to, add to list of extents to delete.
5311          */
5312         if (!(bflags & XFS_BMAPI_REMAP)) {
5313                 if (xfs_is_reflink_inode(ip) && whichfork == XFS_DATA_FORK) {
5314                         xfs_refcount_decrease_extent(tp, del);
5315                 } else if (xfs_ifork_is_realtime(ip, whichfork)) {
5316                         /*
5317                          * Ensure the bitmap and summary inodes are locked
5318                          * and joined to the transaction before modifying them.
5319                          */
5320                         if (!(tp->t_flags & XFS_TRANS_RTBITMAP_LOCKED)) {
5321                                 tp->t_flags |= XFS_TRANS_RTBITMAP_LOCKED;
5322                                 xfs_rtbitmap_lock(tp, mp);
5323                         }
5324                         error = xfs_rtfree_blocks(tp, del->br_startblock,
5325                                         del->br_blockcount);
5326                 } else {
5327                         error = xfs_free_extent_later(tp, del->br_startblock,
5328                                         del->br_blockcount, NULL,
5329                                         XFS_AG_RESV_NONE,
5330                                         ((bflags & XFS_BMAPI_NODISCARD) ||
5331                                         del->br_state == XFS_EXT_UNWRITTEN));
5332                 }
5333                 if (error)
5334                         return error;
5335         }
5336
5337         /*
5338          * Adjust inode # blocks in the file.
5339          */
5340         if (nblks)
5341                 ip->i_nblocks -= nblks;
5342         /*
5343          * Adjust quota data.
5344          */
5345         if (qfield && !(bflags & XFS_BMAPI_REMAP))
5346                 xfs_trans_mod_dquot_byino(tp, ip, qfield, (long)-nblks);
5347
5348         return 0;
5349 }
5350
5351 /*
5352  * Unmap (remove) blocks from a file.
5353  * If nexts is nonzero then the number of extents to remove is limited to
5354  * that value.  If not all extents in the block range can be removed then
5355  * *done is set.
5356  */
5357 static int
5358 __xfs_bunmapi(
5359         struct xfs_trans        *tp,            /* transaction pointer */
5360         struct xfs_inode        *ip,            /* incore inode */
5361         xfs_fileoff_t           start,          /* first file offset deleted */
5362         xfs_filblks_t           *rlen,          /* i/o: amount remaining */
5363         uint32_t                flags,          /* misc flags */
5364         xfs_extnum_t            nexts)          /* number of extents max */
5365 {
5366         struct xfs_btree_cur    *cur;           /* bmap btree cursor */
5367         struct xfs_bmbt_irec    del;            /* extent being deleted */
5368         int                     error;          /* error return value */
5369         xfs_extnum_t            extno;          /* extent number in list */
5370         struct xfs_bmbt_irec    got;            /* current extent record */
5371         struct xfs_ifork        *ifp;           /* inode fork pointer */
5372         int                     isrt;           /* freeing in rt area */
5373         int                     logflags;       /* transaction logging flags */
5374         xfs_extlen_t            mod;            /* rt extent offset */
5375         struct xfs_mount        *mp = ip->i_mount;
5376         int                     tmp_logflags;   /* partial logging flags */
5377         int                     wasdel;         /* was a delayed alloc extent */
5378         int                     whichfork;      /* data or attribute fork */
5379         xfs_filblks_t           len = *rlen;    /* length to unmap in file */
5380         xfs_fileoff_t           end;
5381         struct xfs_iext_cursor  icur;
5382         bool                    done = false;
5383
5384         trace_xfs_bunmap(ip, start, len, flags, _RET_IP_);
5385
5386         whichfork = xfs_bmapi_whichfork(flags);
5387         ASSERT(whichfork != XFS_COW_FORK);
5388         ifp = xfs_ifork_ptr(ip, whichfork);
5389         if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp))) {
5390                 xfs_bmap_mark_sick(ip, whichfork);
5391                 return -EFSCORRUPTED;
5392         }
5393         if (xfs_is_shutdown(mp))
5394                 return -EIO;
5395
5396         xfs_assert_ilocked(ip, XFS_ILOCK_EXCL);
5397         ASSERT(len > 0);
5398         ASSERT(nexts >= 0);
5399
5400         error = xfs_iread_extents(tp, ip, whichfork);
5401         if (error)
5402                 return error;
5403
5404         if (xfs_iext_count(ifp) == 0) {
5405                 *rlen = 0;
5406                 return 0;
5407         }
5408         XFS_STATS_INC(mp, xs_blk_unmap);
5409         isrt = xfs_ifork_is_realtime(ip, whichfork);
5410         end = start + len;
5411
5412         if (!xfs_iext_lookup_extent_before(ip, ifp, &end, &icur, &got)) {
5413                 *rlen = 0;
5414                 return 0;
5415         }
5416         end--;
5417
5418         logflags = 0;
5419         if (ifp->if_format == XFS_DINODE_FMT_BTREE) {
5420                 ASSERT(ifp->if_format == XFS_DINODE_FMT_BTREE);
5421                 cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork);
5422         } else
5423                 cur = NULL;
5424
5425         extno = 0;
5426         while (end != (xfs_fileoff_t)-1 && end >= start &&
5427                (nexts == 0 || extno < nexts)) {
5428                 /*
5429                  * Is the found extent after a hole in which end lives?
5430                  * Just back up to the previous extent, if so.
5431                  */
5432                 if (got.br_startoff > end &&
5433                     !xfs_iext_prev_extent(ifp, &icur, &got)) {
5434                         done = true;
5435                         break;
5436                 }
5437                 /*
5438                  * Is the last block of this extent before the range
5439                  * we're supposed to delete?  If so, we're done.
5440                  */
5441                 end = XFS_FILEOFF_MIN(end,
5442                         got.br_startoff + got.br_blockcount - 1);
5443                 if (end < start)
5444                         break;
5445                 /*
5446                  * Then deal with the (possibly delayed) allocated space
5447                  * we found.
5448                  */
5449                 del = got;
5450                 wasdel = isnullstartblock(del.br_startblock);
5451
5452                 if (got.br_startoff < start) {
5453                         del.br_startoff = start;
5454                         del.br_blockcount -= start - got.br_startoff;
5455                         if (!wasdel)
5456                                 del.br_startblock += start - got.br_startoff;
5457                 }
5458                 if (del.br_startoff + del.br_blockcount > end + 1)
5459                         del.br_blockcount = end + 1 - del.br_startoff;
5460
5461                 if (!isrt || (flags & XFS_BMAPI_REMAP))
5462                         goto delete;
5463
5464                 mod = xfs_rtb_to_rtxoff(mp,
5465                                 del.br_startblock + del.br_blockcount);
5466                 if (mod) {
5467                         /*
5468                          * Realtime extent not lined up at the end.
5469                          * The extent could have been split into written
5470                          * and unwritten pieces, or we could just be
5471                          * unmapping part of it.  But we can't really
5472                          * get rid of part of a realtime extent.
5473                          */
5474                         if (del.br_state == XFS_EXT_UNWRITTEN) {
5475                                 /*
5476                                  * This piece is unwritten, or we're not
5477                                  * using unwritten extents.  Skip over it.
5478                                  */
5479                                 ASSERT((flags & XFS_BMAPI_REMAP) || end >= mod);
5480                                 end -= mod > del.br_blockcount ?
5481                                         del.br_blockcount : mod;
5482                                 if (end < got.br_startoff &&
5483                                     !xfs_iext_prev_extent(ifp, &icur, &got)) {
5484                                         done = true;
5485                                         break;
5486                                 }
5487                                 continue;
5488                         }
5489                         /*
5490                          * It's written, turn it unwritten.
5491                          * This is better than zeroing it.
5492                          */
5493                         ASSERT(del.br_state == XFS_EXT_NORM);
5494                         ASSERT(tp->t_blk_res > 0);
5495                         /*
5496                          * If this spans a realtime extent boundary,
5497                          * chop it back to the start of the one we end at.
5498                          */
5499                         if (del.br_blockcount > mod) {
5500                                 del.br_startoff += del.br_blockcount - mod;
5501                                 del.br_startblock += del.br_blockcount - mod;
5502                                 del.br_blockcount = mod;
5503                         }
5504                         del.br_state = XFS_EXT_UNWRITTEN;
5505                         error = xfs_bmap_add_extent_unwritten_real(tp, ip,
5506                                         whichfork, &icur, &cur, &del,
5507                                         &logflags);
5508                         if (error)
5509                                 goto error0;
5510                         goto nodelete;
5511                 }
5512
5513                 mod = xfs_rtb_to_rtxoff(mp, del.br_startblock);
5514                 if (mod) {
5515                         xfs_extlen_t off = mp->m_sb.sb_rextsize - mod;
5516
5517                         /*
5518                          * Realtime extent is lined up at the end but not
5519                          * at the front.  We'll get rid of full extents if
5520                          * we can.
5521                          */
5522                         if (del.br_blockcount > off) {
5523                                 del.br_blockcount -= off;
5524                                 del.br_startoff += off;
5525                                 del.br_startblock += off;
5526                         } else if (del.br_startoff == start &&
5527                                    (del.br_state == XFS_EXT_UNWRITTEN ||
5528                                     tp->t_blk_res == 0)) {
5529                                 /*
5530                                  * Can't make it unwritten.  There isn't
5531                                  * a full extent here so just skip it.
5532                                  */
5533                                 ASSERT(end >= del.br_blockcount);
5534                                 end -= del.br_blockcount;
5535                                 if (got.br_startoff > end &&
5536                                     !xfs_iext_prev_extent(ifp, &icur, &got)) {
5537                                         done = true;
5538                                         break;
5539                                 }
5540                                 continue;
5541                         } else if (del.br_state == XFS_EXT_UNWRITTEN) {
5542                                 struct xfs_bmbt_irec    prev;
5543                                 xfs_fileoff_t           unwrite_start;
5544
5545                                 /*
5546                                  * This one is already unwritten.
5547                                  * It must have a written left neighbor.
5548                                  * Unwrite the killed part of that one and
5549                                  * try again.
5550                                  */
5551                                 if (!xfs_iext_prev_extent(ifp, &icur, &prev))
5552                                         ASSERT(0);
5553                                 ASSERT(prev.br_state == XFS_EXT_NORM);
5554                                 ASSERT(!isnullstartblock(prev.br_startblock));
5555                                 ASSERT(del.br_startblock ==
5556                                        prev.br_startblock + prev.br_blockcount);
5557                                 unwrite_start = max3(start,
5558                                                      del.br_startoff - mod,
5559                                                      prev.br_startoff);
5560                                 mod = unwrite_start - prev.br_startoff;
5561                                 prev.br_startoff = unwrite_start;
5562                                 prev.br_startblock += mod;
5563                                 prev.br_blockcount -= mod;
5564                                 prev.br_state = XFS_EXT_UNWRITTEN;
5565                                 error = xfs_bmap_add_extent_unwritten_real(tp,
5566                                                 ip, whichfork, &icur, &cur,
5567                                                 &prev, &logflags);
5568                                 if (error)
5569                                         goto error0;
5570                                 goto nodelete;
5571                         } else {
5572                                 ASSERT(del.br_state == XFS_EXT_NORM);
5573                                 del.br_state = XFS_EXT_UNWRITTEN;
5574                                 error = xfs_bmap_add_extent_unwritten_real(tp,
5575                                                 ip, whichfork, &icur, &cur,
5576                                                 &del, &logflags);
5577                                 if (error)
5578                                         goto error0;
5579                                 goto nodelete;
5580                         }
5581                 }
5582
5583 delete:
5584                 if (wasdel) {
5585                         error = xfs_bmap_del_extent_delay(ip, whichfork, &icur,
5586                                         &got, &del);
5587                 } else {
5588                         error = xfs_bmap_del_extent_real(ip, tp, &icur, cur,
5589                                         &del, &tmp_logflags, whichfork,
5590                                         flags);
5591                         logflags |= tmp_logflags;
5592                 }
5593
5594                 if (error)
5595                         goto error0;
5596
5597                 end = del.br_startoff - 1;
5598 nodelete:
5599                 /*
5600                  * If not done go on to the next (previous) record.
5601                  */
5602                 if (end != (xfs_fileoff_t)-1 && end >= start) {
5603                         if (!xfs_iext_get_extent(ifp, &icur, &got) ||
5604                             (got.br_startoff > end &&
5605                              !xfs_iext_prev_extent(ifp, &icur, &got))) {
5606                                 done = true;
5607                                 break;
5608                         }
5609                         extno++;
5610                 }
5611         }
5612         if (done || end == (xfs_fileoff_t)-1 || end < start)
5613                 *rlen = 0;
5614         else
5615                 *rlen = end - start + 1;
5616
5617         /*
5618          * Convert to a btree if necessary.
5619          */
5620         if (xfs_bmap_needs_btree(ip, whichfork)) {
5621                 ASSERT(cur == NULL);
5622                 error = xfs_bmap_extents_to_btree(tp, ip, &cur, 0,
5623                                 &tmp_logflags, whichfork);
5624                 logflags |= tmp_logflags;
5625         } else {
5626                 error = xfs_bmap_btree_to_extents(tp, ip, cur, &logflags,
5627                         whichfork);
5628         }
5629
5630 error0:
5631         /*
5632          * Log everything.  Do this after conversion, there's no point in
5633          * logging the extent records if we've converted to btree format.
5634          */
5635         if ((logflags & xfs_ilog_fext(whichfork)) &&
5636             ifp->if_format != XFS_DINODE_FMT_EXTENTS)
5637                 logflags &= ~xfs_ilog_fext(whichfork);
5638         else if ((logflags & xfs_ilog_fbroot(whichfork)) &&
5639                  ifp->if_format != XFS_DINODE_FMT_BTREE)
5640                 logflags &= ~xfs_ilog_fbroot(whichfork);
5641         /*
5642          * Log inode even in the error case, if the transaction
5643          * is dirty we'll need to shut down the filesystem.
5644          */
5645         if (logflags)
5646                 xfs_trans_log_inode(tp, ip, logflags);
5647         if (cur) {
5648                 if (!error)
5649                         cur->bc_bmap.allocated = 0;
5650                 xfs_btree_del_cursor(cur, error);
5651         }
5652         return error;
5653 }
5654
5655 /* Unmap a range of a file. */
5656 int
5657 xfs_bunmapi(
5658         xfs_trans_t             *tp,
5659         struct xfs_inode        *ip,
5660         xfs_fileoff_t           bno,
5661         xfs_filblks_t           len,
5662         uint32_t                flags,
5663         xfs_extnum_t            nexts,
5664         int                     *done)
5665 {
5666         int                     error;
5667
5668         error = __xfs_bunmapi(tp, ip, bno, &len, flags, nexts);
5669         *done = (len == 0);
5670         return error;
5671 }
5672
5673 /*
5674  * Determine whether an extent shift can be accomplished by a merge with the
5675  * extent that precedes the target hole of the shift.
5676  */
5677 STATIC bool
5678 xfs_bmse_can_merge(
5679         struct xfs_bmbt_irec    *left,  /* preceding extent */
5680         struct xfs_bmbt_irec    *got,   /* current extent to shift */
5681         xfs_fileoff_t           shift)  /* shift fsb */
5682 {
5683         xfs_fileoff_t           startoff;
5684
5685         startoff = got->br_startoff - shift;
5686
5687         /*
5688          * The extent, once shifted, must be adjacent in-file and on-disk with
5689          * the preceding extent.
5690          */
5691         if ((left->br_startoff + left->br_blockcount != startoff) ||
5692             (left->br_startblock + left->br_blockcount != got->br_startblock) ||
5693             (left->br_state != got->br_state) ||
5694             (left->br_blockcount + got->br_blockcount > XFS_MAX_BMBT_EXTLEN))
5695                 return false;
5696
5697         return true;
5698 }
5699
5700 /*
5701  * A bmap extent shift adjusts the file offset of an extent to fill a preceding
5702  * hole in the file. If an extent shift would result in the extent being fully
5703  * adjacent to the extent that currently precedes the hole, we can merge with
5704  * the preceding extent rather than do the shift.
5705  *
5706  * This function assumes the caller has verified a shift-by-merge is possible
5707  * with the provided extents via xfs_bmse_can_merge().
5708  */
5709 STATIC int
5710 xfs_bmse_merge(
5711         struct xfs_trans                *tp,
5712         struct xfs_inode                *ip,
5713         int                             whichfork,
5714         xfs_fileoff_t                   shift,          /* shift fsb */
5715         struct xfs_iext_cursor          *icur,
5716         struct xfs_bmbt_irec            *got,           /* extent to shift */
5717         struct xfs_bmbt_irec            *left,          /* preceding extent */
5718         struct xfs_btree_cur            *cur,
5719         int                             *logflags)      /* output */
5720 {
5721         struct xfs_ifork                *ifp = xfs_ifork_ptr(ip, whichfork);
5722         struct xfs_bmbt_irec            new;
5723         xfs_filblks_t                   blockcount;
5724         int                             error, i;
5725         struct xfs_mount                *mp = ip->i_mount;
5726
5727         blockcount = left->br_blockcount + got->br_blockcount;
5728
5729         xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL);
5730         ASSERT(xfs_bmse_can_merge(left, got, shift));
5731
5732         new = *left;
5733         new.br_blockcount = blockcount;
5734
5735         /*
5736          * Update the on-disk extent count, the btree if necessary and log the
5737          * inode.
5738          */
5739         ifp->if_nextents--;
5740         *logflags |= XFS_ILOG_CORE;
5741         if (!cur) {
5742                 *logflags |= XFS_ILOG_DEXT;
5743                 goto done;
5744         }
5745
5746         /* lookup and remove the extent to merge */
5747         error = xfs_bmbt_lookup_eq(cur, got, &i);
5748         if (error)
5749                 return error;
5750         if (XFS_IS_CORRUPT(mp, i != 1)) {
5751                 xfs_btree_mark_sick(cur);
5752                 return -EFSCORRUPTED;
5753         }
5754
5755         error = xfs_btree_delete(cur, &i);
5756         if (error)
5757                 return error;
5758         if (XFS_IS_CORRUPT(mp, i != 1)) {
5759                 xfs_btree_mark_sick(cur);
5760                 return -EFSCORRUPTED;
5761         }
5762
5763         /* lookup and update size of the previous extent */
5764         error = xfs_bmbt_lookup_eq(cur, left, &i);
5765         if (error)
5766                 return error;
5767         if (XFS_IS_CORRUPT(mp, i != 1)) {
5768                 xfs_btree_mark_sick(cur);
5769                 return -EFSCORRUPTED;
5770         }
5771
5772         error = xfs_bmbt_update(cur, &new);
5773         if (error)
5774                 return error;
5775
5776         /* change to extent format if required after extent removal */
5777         error = xfs_bmap_btree_to_extents(tp, ip, cur, logflags, whichfork);
5778         if (error)
5779                 return error;
5780
5781 done:
5782         xfs_iext_remove(ip, icur, 0);
5783         xfs_iext_prev(ifp, icur);
5784         xfs_iext_update_extent(ip, xfs_bmap_fork_to_state(whichfork), icur,
5785                         &new);
5786
5787         /* update reverse mapping. rmap functions merge the rmaps for us */
5788         xfs_rmap_unmap_extent(tp, ip, whichfork, got);
5789         memcpy(&new, got, sizeof(new));
5790         new.br_startoff = left->br_startoff + left->br_blockcount;
5791         xfs_rmap_map_extent(tp, ip, whichfork, &new);
5792         return 0;
5793 }
5794
5795 static int
5796 xfs_bmap_shift_update_extent(
5797         struct xfs_trans        *tp,
5798         struct xfs_inode        *ip,
5799         int                     whichfork,
5800         struct xfs_iext_cursor  *icur,
5801         struct xfs_bmbt_irec    *got,
5802         struct xfs_btree_cur    *cur,
5803         int                     *logflags,
5804         xfs_fileoff_t           startoff)
5805 {
5806         struct xfs_mount        *mp = ip->i_mount;
5807         struct xfs_bmbt_irec    prev = *got;
5808         int                     error, i;
5809
5810         *logflags |= XFS_ILOG_CORE;
5811
5812         got->br_startoff = startoff;
5813
5814         if (cur) {
5815                 error = xfs_bmbt_lookup_eq(cur, &prev, &i);
5816                 if (error)
5817                         return error;
5818                 if (XFS_IS_CORRUPT(mp, i != 1)) {
5819                         xfs_btree_mark_sick(cur);
5820                         return -EFSCORRUPTED;
5821                 }
5822
5823                 error = xfs_bmbt_update(cur, got);
5824                 if (error)
5825                         return error;
5826         } else {
5827                 *logflags |= XFS_ILOG_DEXT;
5828         }
5829
5830         xfs_iext_update_extent(ip, xfs_bmap_fork_to_state(whichfork), icur,
5831                         got);
5832
5833         /* update reverse mapping */
5834         xfs_rmap_unmap_extent(tp, ip, whichfork, &prev);
5835         xfs_rmap_map_extent(tp, ip, whichfork, got);
5836         return 0;
5837 }
5838
5839 int
5840 xfs_bmap_collapse_extents(
5841         struct xfs_trans        *tp,
5842         struct xfs_inode        *ip,
5843         xfs_fileoff_t           *next_fsb,
5844         xfs_fileoff_t           offset_shift_fsb,
5845         bool                    *done)
5846 {
5847         int                     whichfork = XFS_DATA_FORK;
5848         struct xfs_mount        *mp = ip->i_mount;
5849         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
5850         struct xfs_btree_cur    *cur = NULL;
5851         struct xfs_bmbt_irec    got, prev;
5852         struct xfs_iext_cursor  icur;
5853         xfs_fileoff_t           new_startoff;
5854         int                     error = 0;
5855         int                     logflags = 0;
5856
5857         if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) ||
5858             XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) {
5859                 xfs_bmap_mark_sick(ip, whichfork);
5860                 return -EFSCORRUPTED;
5861         }
5862
5863         if (xfs_is_shutdown(mp))
5864                 return -EIO;
5865
5866         xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL);
5867
5868         error = xfs_iread_extents(tp, ip, whichfork);
5869         if (error)
5870                 return error;
5871
5872         if (ifp->if_format == XFS_DINODE_FMT_BTREE)
5873                 cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork);
5874
5875         if (!xfs_iext_lookup_extent(ip, ifp, *next_fsb, &icur, &got)) {
5876                 *done = true;
5877                 goto del_cursor;
5878         }
5879         if (XFS_IS_CORRUPT(mp, isnullstartblock(got.br_startblock))) {
5880                 xfs_bmap_mark_sick(ip, whichfork);
5881                 error = -EFSCORRUPTED;
5882                 goto del_cursor;
5883         }
5884
5885         new_startoff = got.br_startoff - offset_shift_fsb;
5886         if (xfs_iext_peek_prev_extent(ifp, &icur, &prev)) {
5887                 if (new_startoff < prev.br_startoff + prev.br_blockcount) {
5888                         error = -EINVAL;
5889                         goto del_cursor;
5890                 }
5891
5892                 if (xfs_bmse_can_merge(&prev, &got, offset_shift_fsb)) {
5893                         error = xfs_bmse_merge(tp, ip, whichfork,
5894                                         offset_shift_fsb, &icur, &got, &prev,
5895                                         cur, &logflags);
5896                         if (error)
5897                                 goto del_cursor;
5898                         goto done;
5899                 }
5900         } else {
5901                 if (got.br_startoff < offset_shift_fsb) {
5902                         error = -EINVAL;
5903                         goto del_cursor;
5904                 }
5905         }
5906
5907         error = xfs_bmap_shift_update_extent(tp, ip, whichfork, &icur, &got,
5908                         cur, &logflags, new_startoff);
5909         if (error)
5910                 goto del_cursor;
5911
5912 done:
5913         if (!xfs_iext_next_extent(ifp, &icur, &got)) {
5914                 *done = true;
5915                 goto del_cursor;
5916         }
5917
5918         *next_fsb = got.br_startoff;
5919 del_cursor:
5920         if (cur)
5921                 xfs_btree_del_cursor(cur, error);
5922         if (logflags)
5923                 xfs_trans_log_inode(tp, ip, logflags);
5924         return error;
5925 }
5926
5927 /* Make sure we won't be right-shifting an extent past the maximum bound. */
5928 int
5929 xfs_bmap_can_insert_extents(
5930         struct xfs_inode        *ip,
5931         xfs_fileoff_t           off,
5932         xfs_fileoff_t           shift)
5933 {
5934         struct xfs_bmbt_irec    got;
5935         int                     is_empty;
5936         int                     error = 0;
5937
5938         xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL);
5939
5940         if (xfs_is_shutdown(ip->i_mount))
5941                 return -EIO;
5942
5943         xfs_ilock(ip, XFS_ILOCK_EXCL);
5944         error = xfs_bmap_last_extent(NULL, ip, XFS_DATA_FORK, &got, &is_empty);
5945         if (!error && !is_empty && got.br_startoff >= off &&
5946             ((got.br_startoff + shift) & BMBT_STARTOFF_MASK) < got.br_startoff)
5947                 error = -EINVAL;
5948         xfs_iunlock(ip, XFS_ILOCK_EXCL);
5949
5950         return error;
5951 }
5952
5953 int
5954 xfs_bmap_insert_extents(
5955         struct xfs_trans        *tp,
5956         struct xfs_inode        *ip,
5957         xfs_fileoff_t           *next_fsb,
5958         xfs_fileoff_t           offset_shift_fsb,
5959         bool                    *done,
5960         xfs_fileoff_t           stop_fsb)
5961 {
5962         int                     whichfork = XFS_DATA_FORK;
5963         struct xfs_mount        *mp = ip->i_mount;
5964         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
5965         struct xfs_btree_cur    *cur = NULL;
5966         struct xfs_bmbt_irec    got, next;
5967         struct xfs_iext_cursor  icur;
5968         xfs_fileoff_t           new_startoff;
5969         int                     error = 0;
5970         int                     logflags = 0;
5971
5972         if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) ||
5973             XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) {
5974                 xfs_bmap_mark_sick(ip, whichfork);
5975                 return -EFSCORRUPTED;
5976         }
5977
5978         if (xfs_is_shutdown(mp))
5979                 return -EIO;
5980
5981         xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL);
5982
5983         error = xfs_iread_extents(tp, ip, whichfork);
5984         if (error)
5985                 return error;
5986
5987         if (ifp->if_format == XFS_DINODE_FMT_BTREE)
5988                 cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork);
5989
5990         if (*next_fsb == NULLFSBLOCK) {
5991                 xfs_iext_last(ifp, &icur);
5992                 if (!xfs_iext_get_extent(ifp, &icur, &got) ||
5993                     stop_fsb > got.br_startoff) {
5994                         *done = true;
5995                         goto del_cursor;
5996                 }
5997         } else {
5998                 if (!xfs_iext_lookup_extent(ip, ifp, *next_fsb, &icur, &got)) {
5999                         *done = true;
6000                         goto del_cursor;
6001                 }
6002         }
6003         if (XFS_IS_CORRUPT(mp, isnullstartblock(got.br_startblock))) {
6004                 xfs_bmap_mark_sick(ip, whichfork);
6005                 error = -EFSCORRUPTED;
6006                 goto del_cursor;
6007         }
6008
6009         if (XFS_IS_CORRUPT(mp, stop_fsb > got.br_startoff)) {
6010                 xfs_bmap_mark_sick(ip, whichfork);
6011                 error = -EFSCORRUPTED;
6012                 goto del_cursor;
6013         }
6014
6015         new_startoff = got.br_startoff + offset_shift_fsb;
6016         if (xfs_iext_peek_next_extent(ifp, &icur, &next)) {
6017                 if (new_startoff + got.br_blockcount > next.br_startoff) {
6018                         error = -EINVAL;
6019                         goto del_cursor;
6020                 }
6021
6022                 /*
6023                  * Unlike a left shift (which involves a hole punch), a right
6024                  * shift does not modify extent neighbors in any way.  We should
6025                  * never find mergeable extents in this scenario.  Check anyways
6026                  * and warn if we encounter two extents that could be one.
6027                  */
6028                 if (xfs_bmse_can_merge(&got, &next, offset_shift_fsb))
6029                         WARN_ON_ONCE(1);
6030         }
6031
6032         error = xfs_bmap_shift_update_extent(tp, ip, whichfork, &icur, &got,
6033                         cur, &logflags, new_startoff);
6034         if (error)
6035                 goto del_cursor;
6036
6037         if (!xfs_iext_prev_extent(ifp, &icur, &got) ||
6038             stop_fsb >= got.br_startoff + got.br_blockcount) {
6039                 *done = true;
6040                 goto del_cursor;
6041         }
6042
6043         *next_fsb = got.br_startoff;
6044 del_cursor:
6045         if (cur)
6046                 xfs_btree_del_cursor(cur, error);
6047         if (logflags)
6048                 xfs_trans_log_inode(tp, ip, logflags);
6049         return error;
6050 }
6051
6052 /*
6053  * Splits an extent into two extents at split_fsb block such that it is the
6054  * first block of the current_ext. @ext is a target extent to be split.
6055  * @split_fsb is a block where the extents is split.  If split_fsb lies in a
6056  * hole or the first block of extents, just return 0.
6057  */
6058 int
6059 xfs_bmap_split_extent(
6060         struct xfs_trans        *tp,
6061         struct xfs_inode        *ip,
6062         xfs_fileoff_t           split_fsb)
6063 {
6064         int                             whichfork = XFS_DATA_FORK;
6065         struct xfs_ifork                *ifp = xfs_ifork_ptr(ip, whichfork);
6066         struct xfs_btree_cur            *cur = NULL;
6067         struct xfs_bmbt_irec            got;
6068         struct xfs_bmbt_irec            new; /* split extent */
6069         struct xfs_mount                *mp = ip->i_mount;
6070         xfs_fsblock_t                   gotblkcnt; /* new block count for got */
6071         struct xfs_iext_cursor          icur;
6072         int                             error = 0;
6073         int                             logflags = 0;
6074         int                             i = 0;
6075
6076         if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) ||
6077             XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) {
6078                 xfs_bmap_mark_sick(ip, whichfork);
6079                 return -EFSCORRUPTED;
6080         }
6081
6082         if (xfs_is_shutdown(mp))
6083                 return -EIO;
6084
6085         /* Read in all the extents */
6086         error = xfs_iread_extents(tp, ip, whichfork);
6087         if (error)
6088                 return error;
6089
6090         /*
6091          * If there are not extents, or split_fsb lies in a hole we are done.
6092          */
6093         if (!xfs_iext_lookup_extent(ip, ifp, split_fsb, &icur, &got) ||
6094             got.br_startoff >= split_fsb)
6095                 return 0;
6096
6097         gotblkcnt = split_fsb - got.br_startoff;
6098         new.br_startoff = split_fsb;
6099         new.br_startblock = got.br_startblock + gotblkcnt;
6100         new.br_blockcount = got.br_blockcount - gotblkcnt;
6101         new.br_state = got.br_state;
6102
6103         if (ifp->if_format == XFS_DINODE_FMT_BTREE) {
6104                 cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork);
6105                 error = xfs_bmbt_lookup_eq(cur, &got, &i);
6106                 if (error)
6107                         goto del_cursor;
6108                 if (XFS_IS_CORRUPT(mp, i != 1)) {
6109                         xfs_btree_mark_sick(cur);
6110                         error = -EFSCORRUPTED;
6111                         goto del_cursor;
6112                 }
6113         }
6114
6115         got.br_blockcount = gotblkcnt;
6116         xfs_iext_update_extent(ip, xfs_bmap_fork_to_state(whichfork), &icur,
6117                         &got);
6118
6119         logflags = XFS_ILOG_CORE;
6120         if (cur) {
6121                 error = xfs_bmbt_update(cur, &got);
6122                 if (error)
6123                         goto del_cursor;
6124         } else
6125                 logflags |= XFS_ILOG_DEXT;
6126
6127         /* Add new extent */
6128         xfs_iext_next(ifp, &icur);
6129         xfs_iext_insert(ip, &icur, &new, 0);
6130         ifp->if_nextents++;
6131
6132         if (cur) {
6133                 error = xfs_bmbt_lookup_eq(cur, &new, &i);
6134                 if (error)
6135                         goto del_cursor;
6136                 if (XFS_IS_CORRUPT(mp, i != 0)) {
6137                         xfs_btree_mark_sick(cur);
6138                         error = -EFSCORRUPTED;
6139                         goto del_cursor;
6140                 }
6141                 error = xfs_btree_insert(cur, &i);
6142                 if (error)
6143                         goto del_cursor;
6144                 if (XFS_IS_CORRUPT(mp, i != 1)) {
6145                         xfs_btree_mark_sick(cur);
6146                         error = -EFSCORRUPTED;
6147                         goto del_cursor;
6148                 }
6149         }
6150
6151         /*
6152          * Convert to a btree if necessary.
6153          */
6154         if (xfs_bmap_needs_btree(ip, whichfork)) {
6155                 int tmp_logflags; /* partial log flag return val */
6156
6157                 ASSERT(cur == NULL);
6158                 error = xfs_bmap_extents_to_btree(tp, ip, &cur, 0,
6159                                 &tmp_logflags, whichfork);
6160                 logflags |= tmp_logflags;
6161         }
6162
6163 del_cursor:
6164         if (cur) {
6165                 cur->bc_bmap.allocated = 0;
6166                 xfs_btree_del_cursor(cur, error);
6167         }
6168
6169         if (logflags)
6170                 xfs_trans_log_inode(tp, ip, logflags);
6171         return error;
6172 }
6173
6174 /* Record a bmap intent. */
6175 static inline void
6176 __xfs_bmap_add(
6177         struct xfs_trans                *tp,
6178         enum xfs_bmap_intent_type       type,
6179         struct xfs_inode                *ip,
6180         int                             whichfork,
6181         struct xfs_bmbt_irec            *bmap)
6182 {
6183         struct xfs_bmap_intent          *bi;
6184
6185         if ((whichfork != XFS_DATA_FORK && whichfork != XFS_ATTR_FORK) ||
6186             bmap->br_startblock == HOLESTARTBLOCK ||
6187             bmap->br_startblock == DELAYSTARTBLOCK)
6188                 return;
6189
6190         bi = kmem_cache_alloc(xfs_bmap_intent_cache, GFP_KERNEL | __GFP_NOFAIL);
6191         INIT_LIST_HEAD(&bi->bi_list);
6192         bi->bi_type = type;
6193         bi->bi_owner = ip;
6194         bi->bi_whichfork = whichfork;
6195         bi->bi_bmap = *bmap;
6196
6197         xfs_bmap_defer_add(tp, bi);
6198 }
6199
6200 /* Map an extent into a file. */
6201 void
6202 xfs_bmap_map_extent(
6203         struct xfs_trans        *tp,
6204         struct xfs_inode        *ip,
6205         int                     whichfork,
6206         struct xfs_bmbt_irec    *PREV)
6207 {
6208         __xfs_bmap_add(tp, XFS_BMAP_MAP, ip, whichfork, PREV);
6209 }
6210
6211 /* Unmap an extent out of a file. */
6212 void
6213 xfs_bmap_unmap_extent(
6214         struct xfs_trans        *tp,
6215         struct xfs_inode        *ip,
6216         int                     whichfork,
6217         struct xfs_bmbt_irec    *PREV)
6218 {
6219         __xfs_bmap_add(tp, XFS_BMAP_UNMAP, ip, whichfork, PREV);
6220 }
6221
6222 /*
6223  * Process one of the deferred bmap operations.  We pass back the
6224  * btree cursor to maintain our lock on the bmapbt between calls.
6225  */
6226 int
6227 xfs_bmap_finish_one(
6228         struct xfs_trans                *tp,
6229         struct xfs_bmap_intent          *bi)
6230 {
6231         struct xfs_bmbt_irec            *bmap = &bi->bi_bmap;
6232         int                             error = 0;
6233         int                             flags = 0;
6234
6235         if (bi->bi_whichfork == XFS_ATTR_FORK)
6236                 flags |= XFS_BMAPI_ATTRFORK;
6237
6238         ASSERT(tp->t_highest_agno == NULLAGNUMBER);
6239
6240         trace_xfs_bmap_deferred(bi);
6241
6242         if (XFS_TEST_ERROR(false, tp->t_mountp, XFS_ERRTAG_BMAP_FINISH_ONE))
6243                 return -EIO;
6244
6245         switch (bi->bi_type) {
6246         case XFS_BMAP_MAP:
6247                 if (bi->bi_bmap.br_state == XFS_EXT_UNWRITTEN)
6248                         flags |= XFS_BMAPI_PREALLOC;
6249                 error = xfs_bmapi_remap(tp, bi->bi_owner, bmap->br_startoff,
6250                                 bmap->br_blockcount, bmap->br_startblock,
6251                                 flags);
6252                 bmap->br_blockcount = 0;
6253                 break;
6254         case XFS_BMAP_UNMAP:
6255                 error = __xfs_bunmapi(tp, bi->bi_owner, bmap->br_startoff,
6256                                 &bmap->br_blockcount, flags | XFS_BMAPI_REMAP,
6257                                 1);
6258                 break;
6259         default:
6260                 ASSERT(0);
6261                 xfs_bmap_mark_sick(bi->bi_owner, bi->bi_whichfork);
6262                 error = -EFSCORRUPTED;
6263         }
6264
6265         return error;
6266 }
6267
6268 /* Check that an extent does not have invalid flags or bad ranges. */
6269 xfs_failaddr_t
6270 xfs_bmap_validate_extent_raw(
6271         struct xfs_mount        *mp,
6272         bool                    rtfile,
6273         int                     whichfork,
6274         struct xfs_bmbt_irec    *irec)
6275 {
6276         if (!xfs_verify_fileext(mp, irec->br_startoff, irec->br_blockcount))
6277                 return __this_address;
6278
6279         if (rtfile && whichfork == XFS_DATA_FORK) {
6280                 if (!xfs_verify_rtbext(mp, irec->br_startblock,
6281                                            irec->br_blockcount))
6282                         return __this_address;
6283         } else {
6284                 if (!xfs_verify_fsbext(mp, irec->br_startblock,
6285                                            irec->br_blockcount))
6286                         return __this_address;
6287         }
6288         if (irec->br_state != XFS_EXT_NORM && whichfork != XFS_DATA_FORK)
6289                 return __this_address;
6290         return NULL;
6291 }
6292
6293 int __init
6294 xfs_bmap_intent_init_cache(void)
6295 {
6296         xfs_bmap_intent_cache = kmem_cache_create("xfs_bmap_intent",
6297                         sizeof(struct xfs_bmap_intent),
6298                         0, 0, NULL);
6299
6300         return xfs_bmap_intent_cache != NULL ? 0 : -ENOMEM;
6301 }
6302
6303 void
6304 xfs_bmap_intent_destroy_cache(void)
6305 {
6306         kmem_cache_destroy(xfs_bmap_intent_cache);
6307         xfs_bmap_intent_cache = NULL;
6308 }
6309
6310 /* Check that an inode's extent does not have invalid flags or bad ranges. */
6311 xfs_failaddr_t
6312 xfs_bmap_validate_extent(
6313         struct xfs_inode        *ip,
6314         int                     whichfork,
6315         struct xfs_bmbt_irec    *irec)
6316 {
6317         return xfs_bmap_validate_extent_raw(ip->i_mount,
6318                         XFS_IS_REALTIME_INODE(ip), whichfork, irec);
6319 }
6320
6321 /*
6322  * Used in xfs_itruncate_extents().  This is the maximum number of extents
6323  * freed from a file in a single transaction.
6324  */
6325 #define XFS_ITRUNC_MAX_EXTENTS  2
6326
6327 /*
6328  * Unmap every extent in part of an inode's fork.  We don't do any higher level
6329  * invalidation work at all.
6330  */
6331 int
6332 xfs_bunmapi_range(
6333         struct xfs_trans        **tpp,
6334         struct xfs_inode        *ip,
6335         uint32_t                flags,
6336         xfs_fileoff_t           startoff,
6337         xfs_fileoff_t           endoff)
6338 {
6339         xfs_filblks_t           unmap_len = endoff - startoff + 1;
6340         int                     error = 0;
6341
6342         xfs_assert_ilocked(ip, XFS_ILOCK_EXCL);
6343
6344         while (unmap_len > 0) {
6345                 ASSERT((*tpp)->t_highest_agno == NULLAGNUMBER);
6346                 error = __xfs_bunmapi(*tpp, ip, startoff, &unmap_len, flags,
6347                                 XFS_ITRUNC_MAX_EXTENTS);
6348                 if (error)
6349                         goto out;
6350
6351                 /* free the just unmapped extents */
6352                 error = xfs_defer_finish(tpp);
6353                 if (error)
6354                         goto out;
6355         }
6356 out:
6357         return error;
6358 }
6359
6360 struct xfs_bmap_query_range {
6361         xfs_bmap_query_range_fn fn;
6362         void                    *priv;
6363 };
6364
6365 /* Format btree record and pass to our callback. */
6366 STATIC int
6367 xfs_bmap_query_range_helper(
6368         struct xfs_btree_cur            *cur,
6369         const union xfs_btree_rec       *rec,
6370         void                            *priv)
6371 {
6372         struct xfs_bmap_query_range     *query = priv;
6373         struct xfs_bmbt_irec            irec;
6374         xfs_failaddr_t                  fa;
6375
6376         xfs_bmbt_disk_get_all(&rec->bmbt, &irec);
6377         fa = xfs_bmap_validate_extent(cur->bc_ino.ip, cur->bc_ino.whichfork,
6378                         &irec);
6379         if (fa) {
6380                 xfs_btree_mark_sick(cur);
6381                 return xfs_bmap_complain_bad_rec(cur->bc_ino.ip,
6382                                 cur->bc_ino.whichfork, fa, &irec);
6383         }
6384
6385         return query->fn(cur, &irec, query->priv);
6386 }
6387
6388 /* Find all bmaps. */
6389 int
6390 xfs_bmap_query_all(
6391         struct xfs_btree_cur            *cur,
6392         xfs_bmap_query_range_fn         fn,
6393         void                            *priv)
6394 {
6395         struct xfs_bmap_query_range     query = {
6396                 .priv                   = priv,
6397                 .fn                     = fn,
6398         };
6399
6400         return xfs_btree_query_all(cur, xfs_bmap_query_range_helper, &query);
6401 }