fs/btrfs/inode.c

   1 /*
   2  * Copyright (C) 2007 Oracle.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18
  19 #include <linux/kernel.h>
  20 #include <linux/bio.h>
  21 #include <linux/buffer_head.h>
  22 #include <linux/file.h>
  23 #include <linux/fs.h>
  24 #include <linux/pagemap.h>
  25 #include <linux/highmem.h>
  26 #include <linux/time.h>
  27 #include <linux/init.h>
  28 #include <linux/string.h>
  29 #include <linux/backing-dev.h>
  30 #include <linux/mpage.h>
  31 #include <linux/swap.h>
  32 #include <linux/writeback.h>
  33 #include <linux/compat.h>
  34 #include <linux/bit_spinlock.h>
  35 #include <linux/xattr.h>
  36 #include <linux/posix_acl.h>
  37 #include <linux/falloc.h>
  38 #include <linux/slab.h>
  39 #include <linux/ratelimit.h>
  40 #include <linux/mount.h>
  41 #include <linux/btrfs.h>
  42 #include <linux/blkdev.h>
  43 #include <linux/posix_acl_xattr.h>
  44 #include <linux/uio.h>
  45 #include "ctree.h"
  46 #include "disk-io.h"
  47 #include "transaction.h"
  48 #include "btrfs_inode.h"
  49 #include "print-tree.h"
  50 #include "ordered-data.h"
  51 #include "xattr.h"
  52 #include "tree-log.h"
  53 #include "volumes.h"
  54 #include "compression.h"
  55 #include "locking.h"
  56 #include "free-space-cache.h"
  57 #include "inode-map.h"
  58 #include "backref.h"
  59 #include "hash.h"
  60 #include "props.h"
  61 #include "qgroup.h"
  62 #include "dedupe.h"
  63
  64 struct btrfs_iget_args {
  65         struct btrfs_key *location;
  66         struct btrfs_root *root;
  67 };
  68
  69 struct btrfs_dio_data {
  70         u64 outstanding_extents;
  71         u64 reserve;
  72         u64 unsubmitted_oe_range_start;
  73         u64 unsubmitted_oe_range_end;
  74         int overwrite;
  75 };
  76
  77 static const struct inode_operations btrfs_dir_inode_operations;
  78 static const struct inode_operations btrfs_symlink_inode_operations;
  79 static const struct inode_operations btrfs_dir_ro_inode_operations;
  80 static const struct inode_operations btrfs_special_inode_operations;
  81 static const struct inode_operations btrfs_file_inode_operations;
  82 static const struct address_space_operations btrfs_aops;
  83 static const struct address_space_operations btrfs_symlink_aops;
  84 static const struct file_operations btrfs_dir_file_operations;
  85 static const struct extent_io_ops btrfs_extent_io_ops;
  86
  87 static struct kmem_cache *btrfs_inode_cachep;
  88 struct kmem_cache *btrfs_trans_handle_cachep;
  89 struct kmem_cache *btrfs_path_cachep;
  90 struct kmem_cache *btrfs_free_space_cachep;
  91
  92 #define S_SHIFT 12
  93 static const unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = {
  94         [S_IFREG >> S_SHIFT]    = BTRFS_FT_REG_FILE,
  95         [S_IFDIR >> S_SHIFT]    = BTRFS_FT_DIR,
  96         [S_IFCHR >> S_SHIFT]    = BTRFS_FT_CHRDEV,
  97         [S_IFBLK >> S_SHIFT]    = BTRFS_FT_BLKDEV,
  98         [S_IFIFO >> S_SHIFT]    = BTRFS_FT_FIFO,
  99         [S_IFSOCK >> S_SHIFT]   = BTRFS_FT_SOCK,
 100         [S_IFLNK >> S_SHIFT]    = BTRFS_FT_SYMLINK,
 101 };
 102
 103 static int btrfs_setsize(struct inode *inode, struct iattr *attr);
 104 static int btrfs_truncate(struct inode *inode);
 105 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
 106 static noinline int cow_file_range(struct inode *inode,
 107                                    struct page *locked_page,
 108                                    u64 start, u64 end, u64 delalloc_end,
 109                                    int *page_started, unsigned long *nr_written,
 110                                    int unlock, struct btrfs_dedupe_hash *hash);
 111 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len,
 112                                        u64 orig_start, u64 block_start,
 113                                        u64 block_len, u64 orig_block_len,
 114                                        u64 ram_bytes, int compress_type,
 115                                        int type);
 116
 117 static void __endio_write_update_ordered(struct inode *inode,
 118                                          const u64 offset, const u64 bytes,
 119                                          const bool uptodate);
 120
 121 /*
 122  * Cleanup all submitted ordered extents in specified range to handle errors
 123  * from the fill_dellaloc() callback.
 124  *
 125  * NOTE: caller must ensure that when an error happens, it can not call
 126  * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING
 127  * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata
 128  * to be released, which we want to happen only when finishing the ordered
 129  * extent (btrfs_finish_ordered_io()). Also note that the caller of the
 130  * fill_delalloc() callback already does proper cleanup for the first page of
 131  * the range, that is, it invokes the callback writepage_end_io_hook() for the
 132  * range of the first page.
 133  */
 134 static inline void btrfs_cleanup_ordered_extents(struct inode *inode,
 135                                                  const u64 offset,
 136                                                  const u64 bytes)
 137 {
 138         return __endio_write_update_ordered(inode, offset + PAGE_SIZE,
 139                                             bytes - PAGE_SIZE, false);
 140 }
 141
 142 static int btrfs_dirty_inode(struct inode *inode);
 143
 144 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
 145 void btrfs_test_inode_set_ops(struct inode *inode)
 146 {
 147         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
 148 }
 149 #endif
 150
 151 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
 152                                      struct inode *inode,  struct inode *dir,
 153                                      const struct qstr *qstr)
 154 {
 155         int err;
 156
 157         err = btrfs_init_acl(trans, inode, dir);
 158         if (!err)
 159                 err = btrfs_xattr_security_init(trans, inode, dir, qstr);
 160         return err;
 161 }
 162
 163 /*
 164  * this does all the hard work for inserting an inline extent into
 165  * the btree.  The caller should have done a btrfs_drop_extents so that
 166  * no overlapping inline items exist in the btree
 167  */
 168 static int insert_inline_extent(struct btrfs_trans_handle *trans,
 169                                 struct btrfs_path *path, int extent_inserted,
 170                                 struct btrfs_root *root, struct inode *inode,
 171                                 u64 start, size_t size, size_t compressed_size,
 172                                 int compress_type,
 173                                 struct page **compressed_pages)
 174 {
 175         struct extent_buffer *leaf;
 176         struct page *page = NULL;
 177         char *kaddr;
 178         unsigned long ptr;
 179         struct btrfs_file_extent_item *ei;
 180         int ret;
 181         size_t cur_size = size;
 182         unsigned long offset;
 183
 184         if (compressed_size && compressed_pages)
 185                 cur_size = compressed_size;
 186
 187         inode_add_bytes(inode, size);
 188
 189         if (!extent_inserted) {
 190                 struct btrfs_key key;
 191                 size_t datasize;
 192
 193                 key.objectid = btrfs_ino(BTRFS_I(inode));
 194                 key.offset = start;
 195                 key.type = BTRFS_EXTENT_DATA_KEY;
 196
 197                 datasize = btrfs_file_extent_calc_inline_size(cur_size);
 198                 path->leave_spinning = 1;
 199                 ret = btrfs_insert_empty_item(trans, root, path, &key,
 200                                               datasize);
 201                 if (ret)
 202                         goto fail;
 203         }
 204         leaf = path->nodes[0];
 205         ei = btrfs_item_ptr(leaf, path->slots[0],
 206                             struct btrfs_file_extent_item);
 207         btrfs_set_file_extent_generation(leaf, ei, trans->transid);
 208         btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
 209         btrfs_set_file_extent_encryption(leaf, ei, 0);
 210         btrfs_set_file_extent_other_encoding(leaf, ei, 0);
 211         btrfs_set_file_extent_ram_bytes(leaf, ei, size);
 212         ptr = btrfs_file_extent_inline_start(ei);
 213
 214         if (compress_type != BTRFS_COMPRESS_NONE) {
 215                 struct page *cpage;
 216                 int i = 0;
 217                 while (compressed_size > 0) {
 218                         cpage = compressed_pages[i];
 219                         cur_size = min_t(unsigned long, compressed_size,
 220                                        PAGE_SIZE);
 221
 222                         kaddr = kmap_atomic(cpage);
 223                         write_extent_buffer(leaf, kaddr, ptr, cur_size);
 224                         kunmap_atomic(kaddr);
 225
 226                         i++;
 227                         ptr += cur_size;
 228                         compressed_size -= cur_size;
 229                 }
 230                 btrfs_set_file_extent_compression(leaf, ei,
 231                                                   compress_type);
 232         } else {
 233                 page = find_get_page(inode->i_mapping,
 234                                      start >> PAGE_SHIFT);
 235                 btrfs_set_file_extent_compression(leaf, ei, 0);
 236                 kaddr = kmap_atomic(page);
 237                 offset = start & (PAGE_SIZE - 1);
 238                 write_extent_buffer(leaf, kaddr + offset, ptr, size);
 239                 kunmap_atomic(kaddr);
 240                 put_page(page);
 241         }
 242         btrfs_mark_buffer_dirty(leaf);
 243         btrfs_release_path(path);
 244
 245         /*
 246          * we're an inline extent, so nobody can
 247          * extend the file past i_size without locking
 248          * a page we already have locked.
 249          *
 250          * We must do any isize and inode updates
 251          * before we unlock the pages.  Otherwise we
 252          * could end up racing with unlink.
 253          */
 254         BTRFS_I(inode)->disk_i_size = inode->i_size;
 255         ret = btrfs_update_inode(trans, root, inode);
 256
 257 fail:
 258         return ret;
 259 }
 260
 261
 262 /*
 263  * conditionally insert an inline extent into the file.  This
 264  * does the checks required to make sure the data is small enough
 265  * to fit as an inline extent.
 266  */
 267 static noinline int cow_file_range_inline(struct btrfs_root *root,
 268                                           struct inode *inode, u64 start,
 269                                           u64 end, size_t compressed_size,
 270                                           int compress_type,
 271                                           struct page **compressed_pages)
 272 {
 273         struct btrfs_fs_info *fs_info = root->fs_info;
 274         struct btrfs_trans_handle *trans;
 275         u64 isize = i_size_read(inode);
 276         u64 actual_end = min(end + 1, isize);
 277         u64 inline_len = actual_end - start;
 278         u64 aligned_end = ALIGN(end, fs_info->sectorsize);
 279         u64 data_len = inline_len;
 280         int ret;
 281         struct btrfs_path *path;
 282         int extent_inserted = 0;
 283         u32 extent_item_size;
 284
 285         if (compressed_size)
 286                 data_len = compressed_size;
 287
 288         if (start > 0 ||
 289             actual_end > fs_info->sectorsize ||
 290             data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) ||
 291             (!compressed_size &&
 292             (actual_end & (fs_info->sectorsize - 1)) == 0) ||
 293             end + 1 < isize ||
 294             data_len > fs_info->max_inline) {
 295                 return 1;
 296         }
 297
 298         path = btrfs_alloc_path();
 299         if (!path)
 300                 return -ENOMEM;
 301
 302         trans = btrfs_join_transaction(root);
 303         if (IS_ERR(trans)) {
 304                 btrfs_free_path(path);
 305                 return PTR_ERR(trans);
 306         }
 307         trans->block_rsv = &fs_info->delalloc_block_rsv;
 308
 309         if (compressed_size && compressed_pages)
 310                 extent_item_size = btrfs_file_extent_calc_inline_size(
 311                    compressed_size);
 312         else
 313                 extent_item_size = btrfs_file_extent_calc_inline_size(
 314                     inline_len);
 315
 316         ret = __btrfs_drop_extents(trans, root, inode, path,
 317                                    start, aligned_end, NULL,
 318                                    1, 1, extent_item_size, &extent_inserted);
 319         if (ret) {
 320                 btrfs_abort_transaction(trans, ret);
 321                 goto out;
 322         }
 323
 324         if (isize > actual_end)
 325                 inline_len = min_t(u64, isize, actual_end);
 326         ret = insert_inline_extent(trans, path, extent_inserted,
 327                                    root, inode, start,
 328                                    inline_len, compressed_size,
 329                                    compress_type, compressed_pages);
 330         if (ret && ret != -ENOSPC) {
 331                 btrfs_abort_transaction(trans, ret);
 332                 goto out;
 333         } else if (ret == -ENOSPC) {
 334                 ret = 1;
 335                 goto out;
 336         }
 337
 338         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
 339         btrfs_delalloc_release_metadata(BTRFS_I(inode), end + 1 - start);
 340         btrfs_drop_extent_cache(BTRFS_I(inode), start, aligned_end - 1, 0);
 341 out:
 342         /*
 343          * Don't forget to free the reserved space, as for inlined extent
 344          * it won't count as data extent, free them directly here.
 345          * And at reserve time, it's always aligned to page size, so
 346          * just free one page here.
 347          */
 348         btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE);
 349         btrfs_free_path(path);
 350         btrfs_end_transaction(trans);
 351         return ret;
 352 }
 353
 354 struct async_extent {
 355         u64 start;
 356         u64 ram_size;
 357         u64 compressed_size;
 358         struct page **pages;
 359         unsigned long nr_pages;
 360         int compress_type;
 361         struct list_head list;
 362 };
 363
 364 struct async_cow {
 365         struct inode *inode;
 366         struct btrfs_root *root;
 367         struct page *locked_page;
 368         u64 start;
 369         u64 end;
 370         struct list_head extents;
 371         struct btrfs_work work;
 372 };
 373
 374 static noinline int add_async_extent(struct async_cow *cow,
 375                                      u64 start, u64 ram_size,
 376                                      u64 compressed_size,
 377                                      struct page **pages,
 378                                      unsigned long nr_pages,
 379                                      int compress_type)
 380 {
 381         struct async_extent *async_extent;
 382
 383         async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
 384         BUG_ON(!async_extent); /* -ENOMEM */
 385         async_extent->start = start;
 386         async_extent->ram_size = ram_size;
 387         async_extent->compressed_size = compressed_size;
 388         async_extent->pages = pages;
 389         async_extent->nr_pages = nr_pages;
 390         async_extent->compress_type = compress_type;
 391         list_add_tail(&async_extent->list, &cow->extents);
 392         return 0;
 393 }
 394
 395 static inline int inode_need_compress(struct inode *inode)
 396 {
 397         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 398
 399         /* force compress */
 400         if (btrfs_test_opt(fs_info, FORCE_COMPRESS))
 401                 return 1;
 402         /* bad compression ratios */
 403         if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS)
 404                 return 0;
 405         if (btrfs_test_opt(fs_info, COMPRESS) ||
 406             BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS ||
 407             BTRFS_I(inode)->force_compress)
 408                 return 1;
 409         return 0;
 410 }
 411
 412 static inline void inode_should_defrag(struct btrfs_inode *inode,
 413                 u64 start, u64 end, u64 num_bytes, u64 small_write)
 414 {
 415         /* If this is a small write inside eof, kick off a defrag */
 416         if (num_bytes < small_write &&
 417             (start > 0 || end + 1 < inode->disk_i_size))
 418                 btrfs_add_inode_defrag(NULL, inode);
 419 }
 420
 421 /*
 422  * we create compressed extents in two phases.  The first
 423  * phase compresses a range of pages that have already been
 424  * locked (both pages and state bits are locked).
 425  *
 426  * This is done inside an ordered work queue, and the compression
 427  * is spread across many cpus.  The actual IO submission is step
 428  * two, and the ordered work queue takes care of making sure that
 429  * happens in the same order things were put onto the queue by
 430  * writepages and friends.
 431  *
 432  * If this code finds it can't get good compression, it puts an
 433  * entry onto the work queue to write the uncompressed bytes.  This
 434  * makes sure that both compressed inodes and uncompressed inodes
 435  * are written in the same order that the flusher thread sent them
 436  * down.
 437  */
 438 static noinline void compress_file_range(struct inode *inode,
 439                                         struct page *locked_page,
 440                                         u64 start, u64 end,
 441                                         struct async_cow *async_cow,
 442                                         int *num_added)
 443 {
 444         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 445         struct btrfs_root *root = BTRFS_I(inode)->root;
 446         u64 num_bytes;
 447         u64 blocksize = fs_info->sectorsize;
 448         u64 actual_end;
 449         u64 isize = i_size_read(inode);
 450         int ret = 0;
 451         struct page **pages = NULL;
 452         unsigned long nr_pages;
 453         unsigned long total_compressed = 0;
 454         unsigned long total_in = 0;
 455         int i;
 456         int will_compress;
 457         int compress_type = fs_info->compress_type;
 458         int redirty = 0;
 459
 460         inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1,
 461                         SZ_16K);
 462
 463         actual_end = min_t(u64, isize, end + 1);
 464 again:
 465         will_compress = 0;
 466         nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
 467         BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0);
 468         nr_pages = min_t(unsigned long, nr_pages,
 469                         BTRFS_MAX_COMPRESSED / PAGE_SIZE);
 470
 471         /*
 472          * we don't want to send crud past the end of i_size through
 473          * compression, that's just a waste of CPU time.  So, if the
 474          * end of the file is before the start of our current
 475          * requested range of bytes, we bail out to the uncompressed
 476          * cleanup code that can deal with all of this.
 477          *
 478          * It isn't really the fastest way to fix things, but this is a
 479          * very uncommon corner.
 480          */
 481         if (actual_end <= start)
 482                 goto cleanup_and_bail_uncompressed;
 483
 484         total_compressed = actual_end - start;
 485
 486         /*
 487          * skip compression for a small file range(<=blocksize) that
 488          * isn't an inline extent, since it doesn't save disk space at all.
 489          */
 490         if (total_compressed <= blocksize &&
 491            (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
 492                 goto cleanup_and_bail_uncompressed;
 493
 494         total_compressed = min_t(unsigned long, total_compressed,
 495                         BTRFS_MAX_UNCOMPRESSED);
 496         num_bytes = ALIGN(end - start + 1, blocksize);
 497         num_bytes = max(blocksize,  num_bytes);
 498         total_in = 0;
 499         ret = 0;
 500
 501         /*
 502          * we do compression for mount -o compress and when the
 503          * inode has not been flagged as nocompress.  This flag can
 504          * change at any time if we discover bad compression ratios.
 505          */
 506         if (inode_need_compress(inode)) {
 507                 WARN_ON(pages);
 508                 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);
 509                 if (!pages) {
 510                         /* just bail out to the uncompressed code */
 511                         goto cont;
 512                 }
 513
 514                 if (BTRFS_I(inode)->force_compress)
 515                         compress_type = BTRFS_I(inode)->force_compress;
 516
 517                 /*
 518                  * we need to call clear_page_dirty_for_io on each
 519                  * page in the range.  Otherwise applications with the file
 520                  * mmap'd can wander in and change the page contents while
 521                  * we are compressing them.
 522                  *
 523                  * If the compression fails for any reason, we set the pages
 524                  * dirty again later on.
 525                  */
 526                 extent_range_clear_dirty_for_io(inode, start, end);
 527                 redirty = 1;
 528                 ret = btrfs_compress_pages(compress_type,
 529                                            inode->i_mapping, start,
 530                                            pages,
 531                                            &nr_pages,
 532                                            &total_in,
 533                                            &total_compressed);
 534
 535                 if (!ret) {
 536                         unsigned long offset = total_compressed &
 537                                 (PAGE_SIZE - 1);
 538                         struct page *page = pages[nr_pages - 1];
 539                         char *kaddr;
 540
 541                         /* zero the tail end of the last page, we might be
 542                          * sending it down to disk
 543                          */
 544                         if (offset) {
 545                                 kaddr = kmap_atomic(page);
 546                                 memset(kaddr + offset, 0,
 547                                        PAGE_SIZE - offset);
 548                                 kunmap_atomic(kaddr);
 549                         }
 550                         will_compress = 1;
 551                 }
 552         }
 553 cont:
 554         if (start == 0) {
 555                 /* lets try to make an inline extent */
 556                 if (ret || total_in < (actual_end - start)) {
 557                         /* we didn't compress the entire range, try
 558                          * to make an uncompressed inline extent.
 559                          */
 560                         ret = cow_file_range_inline(root, inode, start, end,
 561                                             0, BTRFS_COMPRESS_NONE, NULL);
 562                 } else {
 563                         /* try making a compressed inline extent */
 564                         ret = cow_file_range_inline(root, inode, start, end,
 565                                                     total_compressed,
 566                                                     compress_type, pages);
 567                 }
 568                 if (ret <= 0) {
 569                         unsigned long clear_flags = EXTENT_DELALLOC |
 570                                 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG;
 571                         unsigned long page_error_op;
 572
 573                         clear_flags |= (ret < 0) ? EXTENT_DO_ACCOUNTING : 0;
 574                         page_error_op = ret < 0 ? PAGE_SET_ERROR : 0;
 575
 576                         /*
 577                          * inline extent creation worked or returned error,
 578                          * we don't need to create any more async work items.
 579                          * Unlock and free up our temp pages.
 580                          */
 581                         extent_clear_unlock_delalloc(inode, start, end, end,
 582                                                      NULL, clear_flags,
 583                                                      PAGE_UNLOCK |
 584                                                      PAGE_CLEAR_DIRTY |
 585                                                      PAGE_SET_WRITEBACK |
 586                                                      page_error_op |
 587                                                      PAGE_END_WRITEBACK);
 588                         if (ret == 0)
 589                                 btrfs_free_reserved_data_space_noquota(inode,
 590                                                                start,
 591                                                                end - start + 1);
 592                         goto free_pages_out;
 593                 }
 594         }
 595
 596         if (will_compress) {
 597                 /*
 598                  * we aren't doing an inline extent round the compressed size
 599                  * up to a block size boundary so the allocator does sane
 600                  * things
 601                  */
 602                 total_compressed = ALIGN(total_compressed, blocksize);
 603
 604                 /*
 605                  * one last check to make sure the compression is really a
 606                  * win, compare the page count read with the blocks on disk,
 607                  * compression must free at least one sector size
 608                  */
 609                 total_in = ALIGN(total_in, PAGE_SIZE);
 610                 if (total_compressed + blocksize <= total_in) {
 611                         num_bytes = total_in;
 612                         *num_added += 1;
 613
 614                         /*
 615                          * The async work queues will take care of doing actual
 616                          * allocation on disk for these compressed pages, and
 617                          * will submit them to the elevator.
 618                          */
 619                         add_async_extent(async_cow, start, num_bytes,
 620                                         total_compressed, pages, nr_pages,
 621                                         compress_type);
 622
 623                         if (start + num_bytes < end) {
 624                                 start += num_bytes;
 625                                 pages = NULL;
 626                                 cond_resched();
 627                                 goto again;
 628                         }
 629                         return;
 630                 }
 631         }
 632         if (pages) {
 633                 /*
 634                  * the compression code ran but failed to make things smaller,
 635                  * free any pages it allocated and our page pointer array
 636                  */
 637                 for (i = 0; i < nr_pages; i++) {
 638                         WARN_ON(pages[i]->mapping);
 639                         put_page(pages[i]);
 640                 }
 641                 kfree(pages);
 642                 pages = NULL;
 643                 total_compressed = 0;
 644                 nr_pages = 0;
 645
 646                 /* flag the file so we don't compress in the future */
 647                 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) &&
 648                     !(BTRFS_I(inode)->force_compress)) {
 649                         BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
 650                 }
 651         }
 652 cleanup_and_bail_uncompressed:
 653         /*
 654          * No compression, but we still need to write the pages in the file
 655          * we've been given so far.  redirty the locked page if it corresponds
 656          * to our extent and set things up for the async work queue to run
 657          * cow_file_range to do the normal delalloc dance.
 658          */
 659         if (page_offset(locked_page) >= start &&
 660             page_offset(locked_page) <= end)
 661                 __set_page_dirty_nobuffers(locked_page);
 662                 /* unlocked later on in the async handlers */
 663
 664         if (redirty)
 665                 extent_range_redirty_for_io(inode, start, end);
 666         add_async_extent(async_cow, start, end - start + 1, 0, NULL, 0,
 667                          BTRFS_COMPRESS_NONE);
 668         *num_added += 1;
 669
 670         return;
 671
 672 free_pages_out:
 673         for (i = 0; i < nr_pages; i++) {
 674                 WARN_ON(pages[i]->mapping);
 675                 put_page(pages[i]);
 676         }
 677         kfree(pages);
 678 }
 679
 680 static void free_async_extent_pages(struct async_extent *async_extent)
 681 {
 682         int i;
 683
 684         if (!async_extent->pages)
 685                 return;
 686
 687         for (i = 0; i < async_extent->nr_pages; i++) {
 688                 WARN_ON(async_extent->pages[i]->mapping);
 689                 put_page(async_extent->pages[i]);
 690         }
 691         kfree(async_extent->pages);
 692         async_extent->nr_pages = 0;
 693         async_extent->pages = NULL;
 694 }
 695
 696 /*
 697  * phase two of compressed writeback.  This is the ordered portion
 698  * of the code, which only gets called in the order the work was
 699  * queued.  We walk all the async extents created by compress_file_range
 700  * and send them down to the disk.
 701  */
 702 static noinline void submit_compressed_extents(struct inode *inode,
 703                                               struct async_cow *async_cow)
 704 {
 705         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 706         struct async_extent *async_extent;
 707         u64 alloc_hint = 0;
 708         struct btrfs_key ins;
 709         struct extent_map *em;
 710         struct btrfs_root *root = BTRFS_I(inode)->root;
 711         struct extent_io_tree *io_tree;
 712         int ret = 0;
 713
 714 again:
 715         while (!list_empty(&async_cow->extents)) {
 716                 async_extent = list_entry(async_cow->extents.next,
 717                                           struct async_extent, list);
 718                 list_del(&async_extent->list);
 719
 720                 io_tree = &BTRFS_I(inode)->io_tree;
 721
 722 retry:
 723                 /* did the compression code fall back to uncompressed IO? */
 724                 if (!async_extent->pages) {
 725                         int page_started = 0;
 726                         unsigned long nr_written = 0;
 727
 728                         lock_extent(io_tree, async_extent->start,
 729                                          async_extent->start +
 730                                          async_extent->ram_size - 1);
 731
 732                         /* allocate blocks */
 733                         ret = cow_file_range(inode, async_cow->locked_page,
 734                                              async_extent->start,
 735                                              async_extent->start +
 736                                              async_extent->ram_size - 1,
 737                                              async_extent->start +
 738                                              async_extent->ram_size - 1,
 739                                              &page_started, &nr_written, 0,
 740                                              NULL);
 741
 742                         /* JDM XXX */
 743
 744                         /*
 745                          * if page_started, cow_file_range inserted an
 746                          * inline extent and took care of all the unlocking
 747                          * and IO for us.  Otherwise, we need to submit
 748                          * all those pages down to the drive.
 749                          */
 750                         if (!page_started && !ret)
 751                                 extent_write_locked_range(io_tree,
 752                                                   inode, async_extent->start,
 753                                                   async_extent->start +
 754                                                   async_extent->ram_size - 1,
 755                                                   btrfs_get_extent,
 756                                                   WB_SYNC_ALL);
 757                         else if (ret)
 758                                 unlock_page(async_cow->locked_page);
 759                         kfree(async_extent);
 760                         cond_resched();
 761                         continue;
 762                 }
 763
 764                 lock_extent(io_tree, async_extent->start,
 765                             async_extent->start + async_extent->ram_size - 1);
 766
 767                 ret = btrfs_reserve_extent(root, async_extent->ram_size,
 768                                            async_extent->compressed_size,
 769                                            async_extent->compressed_size,
 770                                            0, alloc_hint, &ins, 1, 1);
 771                 if (ret) {
 772                         free_async_extent_pages(async_extent);
 773
 774                         if (ret == -ENOSPC) {
 775                                 unlock_extent(io_tree, async_extent->start,
 776                                               async_extent->start +
 777                                               async_extent->ram_size - 1);
 778
 779                                 /*
 780                                  * we need to redirty the pages if we decide to
 781                                  * fallback to uncompressed IO, otherwise we
 782                                  * will not submit these pages down to lower
 783                                  * layers.
 784                                  */
 785                                 extent_range_redirty_for_io(inode,
 786                                                 async_extent->start,
 787                                                 async_extent->start +
 788                                                 async_extent->ram_size - 1);
 789
 790                                 goto retry;
 791                         }
 792                         goto out_free;
 793                 }
 794                 /*
 795                  * here we're doing allocation and writeback of the
 796                  * compressed pages
 797                  */
 798                 em = create_io_em(inode, async_extent->start,
 799                                   async_extent->ram_size, /* len */
 800                                   async_extent->start, /* orig_start */
 801                                   ins.objectid, /* block_start */
 802                                   ins.offset, /* block_len */
 803                                   ins.offset, /* orig_block_len */
 804                                   async_extent->ram_size, /* ram_bytes */
 805                                   async_extent->compress_type,
 806                                   BTRFS_ORDERED_COMPRESSED);
 807                 if (IS_ERR(em))
 808                         /* ret value is not necessary due to void function */
 809                         goto out_free_reserve;
 810                 free_extent_map(em);
 811
 812                 ret = btrfs_add_ordered_extent_compress(inode,
 813                                                 async_extent->start,
 814                                                 ins.objectid,
 815                                                 async_extent->ram_size,
 816                                                 ins.offset,
 817                                                 BTRFS_ORDERED_COMPRESSED,
 818                                                 async_extent->compress_type);
 819                 if (ret) {
 820                         btrfs_drop_extent_cache(BTRFS_I(inode),
 821                                                 async_extent->start,
 822                                                 async_extent->start +
 823                                                 async_extent->ram_size - 1, 0);
 824                         goto out_free_reserve;
 825                 }
 826                 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
 827
 828                 /*
 829                  * clear dirty, set writeback and unlock the pages.
 830                  */
 831                 extent_clear_unlock_delalloc(inode, async_extent->start,
 832                                 async_extent->start +
 833                                 async_extent->ram_size - 1,
 834                                 async_extent->start +
 835                                 async_extent->ram_size - 1,
 836                                 NULL, EXTENT_LOCKED | EXTENT_DELALLOC,
 837                                 PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
 838                                 PAGE_SET_WRITEBACK);
 839                 if (btrfs_submit_compressed_write(inode,
 840                                     async_extent->start,
 841                                     async_extent->ram_size,
 842                                     ins.objectid,
 843                                     ins.offset, async_extent->pages,
 844                                     async_extent->nr_pages)) {
 845                         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
 846                         struct page *p = async_extent->pages[0];
 847                         const u64 start = async_extent->start;
 848                         const u64 end = start + async_extent->ram_size - 1;
 849
 850                         p->mapping = inode->i_mapping;
 851                         tree->ops->writepage_end_io_hook(p, start, end,
 852                                                          NULL, 0);
 853                         p->mapping = NULL;
 854                         extent_clear_unlock_delalloc(inode, start, end, end,
 855                                                      NULL, 0,
 856                                                      PAGE_END_WRITEBACK |
 857                                                      PAGE_SET_ERROR);
 858                         free_async_extent_pages(async_extent);
 859                 }
 860                 alloc_hint = ins.objectid + ins.offset;
 861                 kfree(async_extent);
 862                 cond_resched();
 863         }
 864         return;
 865 out_free_reserve:
 866         btrfs_dec_block_group_reservations(fs_info, ins.objectid);
 867         btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
 868 out_free:
 869         extent_clear_unlock_delalloc(inode, async_extent->start,
 870                                      async_extent->start +
 871                                      async_extent->ram_size - 1,
 872                                      async_extent->start +
 873                                      async_extent->ram_size - 1,
 874                                      NULL, EXTENT_LOCKED | EXTENT_DELALLOC |
 875                                      EXTENT_DELALLOC_NEW |
 876                                      EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
 877                                      PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
 878                                      PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK |
 879                                      PAGE_SET_ERROR);
 880         free_async_extent_pages(async_extent);
 881         kfree(async_extent);
 882         goto again;
 883 }
 884
 885 static u64 get_extent_allocation_hint(struct inode *inode, u64 start,
 886                                       u64 num_bytes)
 887 {
 888         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
 889         struct extent_map *em;
 890         u64 alloc_hint = 0;
 891
 892         read_lock(&em_tree->lock);
 893         em = search_extent_mapping(em_tree, start, num_bytes);
 894         if (em) {
 895                 /*
 896                  * if block start isn't an actual block number then find the
 897                  * first block in this inode and use that as a hint.  If that
 898                  * block is also bogus then just don't worry about it.
 899                  */
 900                 if (em->block_start >= EXTENT_MAP_LAST_BYTE) {
 901                         free_extent_map(em);
 902                         em = search_extent_mapping(em_tree, 0, 0);
 903                         if (em && em->block_start < EXTENT_MAP_LAST_BYTE)
 904                                 alloc_hint = em->block_start;
 905                         if (em)
 906                                 free_extent_map(em);
 907                 } else {
 908                         alloc_hint = em->block_start;
 909                         free_extent_map(em);
 910                 }
 911         }
 912         read_unlock(&em_tree->lock);
 913
 914         return alloc_hint;
 915 }
 916
 917 /*
 918  * when extent_io.c finds a delayed allocation range in the file,
 919  * the call backs end up in this code.  The basic idea is to
 920  * allocate extents on disk for the range, and create ordered data structs
 921  * in ram to track those extents.
 922  *
 923  * locked_page is the page that writepage had locked already.  We use
 924  * it to make sure we don't do extra locks or unlocks.
 925  *
 926  * *page_started is set to one if we unlock locked_page and do everything
 927  * required to start IO on it.  It may be clean and already done with
 928  * IO when we return.
 929  */
 930 static noinline int cow_file_range(struct inode *inode,
 931                                    struct page *locked_page,
 932                                    u64 start, u64 end, u64 delalloc_end,
 933                                    int *page_started, unsigned long *nr_written,
 934                                    int unlock, struct btrfs_dedupe_hash *hash)
 935 {
 936         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 937         struct btrfs_root *root = BTRFS_I(inode)->root;
 938         u64 alloc_hint = 0;
 939         u64 num_bytes;
 940         unsigned long ram_size;
 941         u64 disk_num_bytes;
 942         u64 cur_alloc_size = 0;
 943         u64 blocksize = fs_info->sectorsize;
 944         struct btrfs_key ins;
 945         struct extent_map *em;
 946         unsigned clear_bits;
 947         unsigned long page_ops;
 948         bool extent_reserved = false;
 949         int ret = 0;
 950
 951         if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
 952                 WARN_ON_ONCE(1);
 953                 ret = -EINVAL;
 954                 goto out_unlock;
 955         }
 956
 957         num_bytes = ALIGN(end - start + 1, blocksize);
 958         num_bytes = max(blocksize,  num_bytes);
 959         disk_num_bytes = num_bytes;
 960
 961         inode_should_defrag(BTRFS_I(inode), start, end, num_bytes, SZ_64K);
 962
 963         if (start == 0) {
 964                 /* lets try to make an inline extent */
 965                 ret = cow_file_range_inline(root, inode, start, end, 0,
 966                                         BTRFS_COMPRESS_NONE, NULL);
 967                 if (ret == 0) {
 968                         extent_clear_unlock_delalloc(inode, start, end,
 969                                      delalloc_end, NULL,
 970                                      EXTENT_LOCKED | EXTENT_DELALLOC |
 971                                      EXTENT_DELALLOC_NEW |
 972                                      EXTENT_DEFRAG, PAGE_UNLOCK |
 973                                      PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
 974                                      PAGE_END_WRITEBACK);
 975                         btrfs_free_reserved_data_space_noquota(inode, start,
 976                                                 end - start + 1);
 977                         *nr_written = *nr_written +
 978                              (end - start + PAGE_SIZE) / PAGE_SIZE;
 979                         *page_started = 1;
 980                         goto out;
 981                 } else if (ret < 0) {
 982                         goto out_unlock;
 983                 }
 984         }
 985
 986         BUG_ON(disk_num_bytes >
 987                btrfs_super_total_bytes(fs_info->super_copy));
 988
 989         alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
 990         btrfs_drop_extent_cache(BTRFS_I(inode), start,
 991                         start + num_bytes - 1, 0);
 992
 993         while (disk_num_bytes > 0) {
 994                 cur_alloc_size = disk_num_bytes;
 995                 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size,
 996                                            fs_info->sectorsize, 0, alloc_hint,
 997                                            &ins, 1, 1);
 998                 if (ret < 0)
 999                         goto out_unlock;
1000                 cur_alloc_size = ins.offset;
1001                 extent_reserved = true;
1002
1003                 ram_size = ins.offset;
1004                 em = create_io_em(inode, start, ins.offset, /* len */
1005                                   start, /* orig_start */
1006                                   ins.objectid, /* block_start */
1007                                   ins.offset, /* block_len */
1008                                   ins.offset, /* orig_block_len */
1009                                   ram_size, /* ram_bytes */
1010                                   BTRFS_COMPRESS_NONE, /* compress_type */
1011                                   BTRFS_ORDERED_REGULAR /* type */);
1012                 if (IS_ERR(em))
1013                         goto out_reserve;
1014                 free_extent_map(em);
1015
1016                 ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
1017                                                ram_size, cur_alloc_size, 0);
1018                 if (ret)
1019                         goto out_drop_extent_cache;
1020
1021                 if (root->root_key.objectid ==
1022                     BTRFS_DATA_RELOC_TREE_OBJECTID) {
1023                         ret = btrfs_reloc_clone_csums(inode, start,
1024                                                       cur_alloc_size);
1025                         /*
1026                          * Only drop cache here, and process as normal.
1027                          *
1028                          * We must not allow extent_clear_unlock_delalloc()
1029                          * at out_unlock label to free meta of this ordered
1030                          * extent, as its meta should be freed by
1031                          * btrfs_finish_ordered_io().
1032                          *
1033                          * So we must continue until @start is increased to
1034                          * skip current ordered extent.
1035                          */
1036                         if (ret)
1037                                 btrfs_drop_extent_cache(BTRFS_I(inode), start,
1038                                                 start + ram_size - 1, 0);
1039                 }
1040
1041                 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1042
1043                 /* we're not doing compressed IO, don't unlock the first
1044                  * page (which the caller expects to stay locked), don't
1045                  * clear any dirty bits and don't set any writeback bits
1046                  *
1047                  * Do set the Private2 bit so we know this page was properly
1048                  * setup for writepage
1049                  */
1050                 page_ops = unlock ? PAGE_UNLOCK : 0;
1051                 page_ops |= PAGE_SET_PRIVATE2;
1052
1053                 extent_clear_unlock_delalloc(inode, start,
1054                                              start + ram_size - 1,
1055                                              delalloc_end, locked_page,
1056                                              EXTENT_LOCKED | EXTENT_DELALLOC,
1057                                              page_ops);
1058                 if (disk_num_bytes < cur_alloc_size)
1059                         disk_num_bytes = 0;
1060                 else
1061                         disk_num_bytes -= cur_alloc_size;
1062                 num_bytes -= cur_alloc_size;
1063                 alloc_hint = ins.objectid + ins.offset;
1064                 start += cur_alloc_size;
1065                 extent_reserved = false;
1066
1067                 /*
1068                  * btrfs_reloc_clone_csums() error, since start is increased
1069                  * extent_clear_unlock_delalloc() at out_unlock label won't
1070                  * free metadata of current ordered extent, we're OK to exit.
1071                  */
1072                 if (ret)
1073                         goto out_unlock;
1074         }
1075 out:
1076         return ret;
1077
1078 out_drop_extent_cache:
1079         btrfs_drop_extent_cache(BTRFS_I(inode), start, start + ram_size - 1, 0);
1080 out_reserve:
1081         btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1082         btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
1083 out_unlock:
1084         clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
1085                 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV;
1086         page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
1087                 PAGE_END_WRITEBACK;
1088         /*
1089          * If we reserved an extent for our delalloc range (or a subrange) and
1090          * failed to create the respective ordered extent, then it means that
1091          * when we reserved the extent we decremented the extent's size from
1092          * the data space_info's bytes_may_use counter and incremented the
1093          * space_info's bytes_reserved counter by the same amount. We must make
1094          * sure extent_clear_unlock_delalloc() does not try to decrement again
1095          * the data space_info's bytes_may_use counter, therefore we do not pass
1096          * it the flag EXTENT_CLEAR_DATA_RESV.
1097          */
1098         if (extent_reserved) {
1099                 extent_clear_unlock_delalloc(inode, start,
1100                                              start + cur_alloc_size,
1101                                              start + cur_alloc_size,
1102                                              locked_page,
1103                                              clear_bits,
1104                                              page_ops);
1105                 start += cur_alloc_size;
1106                 if (start >= end)
1107                         goto out;
1108         }
1109         extent_clear_unlock_delalloc(inode, start, end, delalloc_end,
1110                                      locked_page,
1111                                      clear_bits | EXTENT_CLEAR_DATA_RESV,
1112                                      page_ops);
1113         goto out;
1114 }
1115
1116 /*
1117  * work queue call back to started compression on a file and pages
1118  */
1119 static noinline void async_cow_start(struct btrfs_work *work)
1120 {
1121         struct async_cow *async_cow;
1122         int num_added = 0;
1123         async_cow = container_of(work, struct async_cow, work);
1124
1125         compress_file_range(async_cow->inode, async_cow->locked_page,
1126                             async_cow->start, async_cow->end, async_cow,
1127                             &num_added);
1128         if (num_added == 0) {
1129                 btrfs_add_delayed_iput(async_cow->inode);
1130                 async_cow->inode = NULL;
1131         }
1132 }
1133
1134 /*
1135  * work queue call back to submit previously compressed pages
1136  */
1137 static noinline void async_cow_submit(struct btrfs_work *work)
1138 {
1139         struct btrfs_fs_info *fs_info;
1140         struct async_cow *async_cow;
1141         struct btrfs_root *root;
1142         unsigned long nr_pages;
1143
1144         async_cow = container_of(work, struct async_cow, work);
1145
1146         root = async_cow->root;
1147         fs_info = root->fs_info;
1148         nr_pages = (async_cow->end - async_cow->start + PAGE_SIZE) >>
1149                 PAGE_SHIFT;
1150
1151         /*
1152          * atomic_sub_return implies a barrier for waitqueue_active
1153          */
1154         if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) <
1155             5 * SZ_1M &&
1156             waitqueue_active(&fs_info->async_submit_wait))
1157                 wake_up(&fs_info->async_submit_wait);
1158
1159         if (async_cow->inode)
1160                 submit_compressed_extents(async_cow->inode, async_cow);
1161 }
1162
1163 static noinline void async_cow_free(struct btrfs_work *work)
1164 {
1165         struct async_cow *async_cow;
1166         async_cow = container_of(work, struct async_cow, work);
1167         if (async_cow->inode)
1168                 btrfs_add_delayed_iput(async_cow->inode);
1169         kfree(async_cow);
1170 }
1171
1172 static int cow_file_range_async(struct inode *inode, struct page *locked_page,
1173                                 u64 start, u64 end, int *page_started,
1174                                 unsigned long *nr_written)
1175 {
1176         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1177         struct async_cow *async_cow;
1178         struct btrfs_root *root = BTRFS_I(inode)->root;
1179         unsigned long nr_pages;
1180         u64 cur_end;
1181
1182         clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED,
1183                          1, 0, NULL, GFP_NOFS);
1184         while (start < end) {
1185                 async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS);
1186                 BUG_ON(!async_cow); /* -ENOMEM */
1187                 async_cow->inode = igrab(inode);
1188                 async_cow->root = root;
1189                 async_cow->locked_page = locked_page;
1190                 async_cow->start = start;
1191
1192                 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS &&
1193                     !btrfs_test_opt(fs_info, FORCE_COMPRESS))
1194                         cur_end = end;
1195                 else
1196                         cur_end = min(end, start + SZ_512K - 1);
1197
1198                 async_cow->end = cur_end;
1199                 INIT_LIST_HEAD(&async_cow->extents);
1200
1201                 btrfs_init_work(&async_cow->work,
1202                                 btrfs_delalloc_helper,
1203                                 async_cow_start, async_cow_submit,
1204                                 async_cow_free);
1205
1206                 nr_pages = (cur_end - start + PAGE_SIZE) >>
1207                         PAGE_SHIFT;
1208                 atomic_add(nr_pages, &fs_info->async_delalloc_pages);
1209
1210                 btrfs_queue_work(fs_info->delalloc_workers, &async_cow->work);
1211
1212                 while (atomic_read(&fs_info->async_submit_draining) &&
1213                        atomic_read(&fs_info->async_delalloc_pages)) {
1214                         wait_event(fs_info->async_submit_wait,
1215                                    (atomic_read(&fs_info->async_delalloc_pages) ==
1216                                     0));
1217                 }
1218
1219                 *nr_written += nr_pages;
1220                 start = cur_end + 1;
1221         }
1222         *page_started = 1;
1223         return 0;
1224 }
1225
1226 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info,
1227                                         u64 bytenr, u64 num_bytes)
1228 {
1229         int ret;
1230         struct btrfs_ordered_sum *sums;
1231         LIST_HEAD(list);
1232
1233         ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr,
1234                                        bytenr + num_bytes - 1, &list, 0);
1235         if (ret == 0 && list_empty(&list))
1236                 return 0;
1237
1238         while (!list_empty(&list)) {
1239                 sums = list_entry(list.next, struct btrfs_ordered_sum, list);
1240                 list_del(&sums->list);
1241                 kfree(sums);
1242         }
1243         return 1;
1244 }
1245
1246 /*
1247  * when nowcow writeback call back.  This checks for snapshots or COW copies
1248  * of the extents that exist in the file, and COWs the file as required.
1249  *
1250  * If no cow copies or snapshots exist, we write directly to the existing
1251  * blocks on disk
1252  */
1253 static noinline int run_delalloc_nocow(struct inode *inode,
1254                                        struct page *locked_page,
1255                               u64 start, u64 end, int *page_started, int force,
1256                               unsigned long *nr_written)
1257 {
1258         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1259         struct btrfs_root *root = BTRFS_I(inode)->root;
1260         struct extent_buffer *leaf;
1261         struct btrfs_path *path;
1262         struct btrfs_file_extent_item *fi;
1263         struct btrfs_key found_key;
1264         struct extent_map *em;
1265         u64 cow_start;
1266         u64 cur_offset;
1267         u64 extent_end;
1268         u64 extent_offset;
1269         u64 disk_bytenr;
1270         u64 num_bytes;
1271         u64 disk_num_bytes;
1272         u64 ram_bytes;
1273         int extent_type;
1274         int ret, err;
1275         int type;
1276         int nocow;
1277         int check_prev = 1;
1278         bool nolock;
1279         u64 ino = btrfs_ino(BTRFS_I(inode));
1280
1281         path = btrfs_alloc_path();
1282         if (!path) {
1283                 extent_clear_unlock_delalloc(inode, start, end, end,
1284                                              locked_page,
1285                                              EXTENT_LOCKED | EXTENT_DELALLOC |
1286                                              EXTENT_DO_ACCOUNTING |
1287                                              EXTENT_DEFRAG, PAGE_UNLOCK |
1288                                              PAGE_CLEAR_DIRTY |
1289                                              PAGE_SET_WRITEBACK |
1290                                              PAGE_END_WRITEBACK);
1291                 return -ENOMEM;
1292         }
1293
1294         nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
1295
1296         cow_start = (u64)-1;
1297         cur_offset = start;
1298         while (1) {
1299                 ret = btrfs_lookup_file_extent(NULL, root, path, ino,
1300                                                cur_offset, 0);
1301                 if (ret < 0)
1302                         goto error;
1303                 if (ret > 0 && path->slots[0] > 0 && check_prev) {
1304                         leaf = path->nodes[0];
1305                         btrfs_item_key_to_cpu(leaf, &found_key,
1306                                               path->slots[0] - 1);
1307                         if (found_key.objectid == ino &&
1308                             found_key.type == BTRFS_EXTENT_DATA_KEY)
1309                                 path->slots[0]--;
1310                 }
1311                 check_prev = 0;
1312 next_slot:
1313                 leaf = path->nodes[0];
1314                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
1315                         ret = btrfs_next_leaf(root, path);
1316                         if (ret < 0)
1317                                 goto error;
1318                         if (ret > 0)
1319                                 break;
1320                         leaf = path->nodes[0];
1321                 }
1322
1323                 nocow = 0;
1324                 disk_bytenr = 0;
1325                 num_bytes = 0;
1326                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1327
1328                 if (found_key.objectid > ino)
1329                         break;
1330                 if (WARN_ON_ONCE(found_key.objectid < ino) ||
1331                     found_key.type < BTRFS_EXTENT_DATA_KEY) {
1332                         path->slots[0]++;
1333                         goto next_slot;
1334                 }
1335                 if (found_key.type > BTRFS_EXTENT_DATA_KEY ||
1336                     found_key.offset > end)
1337                         break;
1338
1339                 if (found_key.offset > cur_offset) {
1340                         extent_end = found_key.offset;
1341                         extent_type = 0;
1342                         goto out_check;
1343                 }
1344
1345                 fi = btrfs_item_ptr(leaf, path->slots[0],
1346                                     struct btrfs_file_extent_item);
1347                 extent_type = btrfs_file_extent_type(leaf, fi);
1348
1349                 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
1350                 if (extent_type == BTRFS_FILE_EXTENT_REG ||
1351                     extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1352                         disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1353                         extent_offset = btrfs_file_extent_offset(leaf, fi);
1354                         extent_end = found_key.offset +
1355                                 btrfs_file_extent_num_bytes(leaf, fi);
1356                         disk_num_bytes =
1357                                 btrfs_file_extent_disk_num_bytes(leaf, fi);
1358                         if (extent_end <= start) {
1359                                 path->slots[0]++;
1360                                 goto next_slot;
1361                         }
1362                         if (disk_bytenr == 0)
1363                                 goto out_check;
1364                         if (btrfs_file_extent_compression(leaf, fi) ||
1365                             btrfs_file_extent_encryption(leaf, fi) ||
1366                             btrfs_file_extent_other_encoding(leaf, fi))
1367                                 goto out_check;
1368                         if (extent_type == BTRFS_FILE_EXTENT_REG && !force)
1369                                 goto out_check;
1370                         if (btrfs_extent_readonly(fs_info, disk_bytenr))
1371                                 goto out_check;
1372                         if (btrfs_cross_ref_exist(root, ino,
1373                                                   found_key.offset -
1374                                                   extent_offset, disk_bytenr))
1375                                 goto out_check;
1376                         disk_bytenr += extent_offset;
1377                         disk_bytenr += cur_offset - found_key.offset;
1378                         num_bytes = min(end + 1, extent_end) - cur_offset;
1379                         /*
1380                          * if there are pending snapshots for this root,
1381                          * we fall into common COW way.
1382                          */
1383                         if (!nolock) {
1384                                 err = btrfs_start_write_no_snapshoting(root);
1385                                 if (!err)
1386                                         goto out_check;
1387                         }
1388                         /*
1389                          * force cow if csum exists in the range.
1390                          * this ensure that csum for a given extent are
1391                          * either valid or do not exist.
1392                          */
1393                         if (csum_exist_in_range(fs_info, disk_bytenr,
1394                                                 num_bytes)) {
1395                                 if (!nolock)
1396                                         btrfs_end_write_no_snapshoting(root);
1397                                 goto out_check;
1398                         }
1399                         if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) {
1400                                 if (!nolock)
1401                                         btrfs_end_write_no_snapshoting(root);
1402                                 goto out_check;
1403                         }
1404                         nocow = 1;
1405                 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
1406                         extent_end = found_key.offset +
1407                                 btrfs_file_extent_inline_len(leaf,
1408                                                      path->slots[0], fi);
1409                         extent_end = ALIGN(extent_end,
1410                                            fs_info->sectorsize);
1411                 } else {
1412                         BUG_ON(1);
1413                 }
1414 out_check:
1415                 if (extent_end <= start) {
1416                         path->slots[0]++;
1417                         if (!nolock && nocow)
1418                                 btrfs_end_write_no_snapshoting(root);
1419                         if (nocow)
1420                                 btrfs_dec_nocow_writers(fs_info, disk_bytenr);
1421                         goto next_slot;
1422                 }
1423                 if (!nocow) {
1424                         if (cow_start == (u64)-1)
1425                                 cow_start = cur_offset;
1426                         cur_offset = extent_end;
1427                         if (cur_offset > end)
1428                                 break;
1429                         path->slots[0]++;
1430                         goto next_slot;
1431                 }
1432
1433                 btrfs_release_path(path);
1434                 if (cow_start != (u64)-1) {
1435                         ret = cow_file_range(inode, locked_page,
1436                                              cow_start, found_key.offset - 1,
1437                                              end, page_started, nr_written, 1,
1438                                              NULL);
1439                         if (ret) {
1440                                 if (!nolock && nocow)
1441                                         btrfs_end_write_no_snapshoting(root);
1442                                 if (nocow)
1443                                         btrfs_dec_nocow_writers(fs_info,
1444                                                                 disk_bytenr);
1445                                 goto error;
1446                         }
1447                         cow_start = (u64)-1;
1448                 }
1449
1450                 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1451                         u64 orig_start = found_key.offset - extent_offset;
1452
1453                         em = create_io_em(inode, cur_offset, num_bytes,
1454                                           orig_start,
1455                                           disk_bytenr, /* block_start */
1456                                           num_bytes, /* block_len */
1457                                           disk_num_bytes, /* orig_block_len */
1458                                           ram_bytes, BTRFS_COMPRESS_NONE,
1459                                           BTRFS_ORDERED_PREALLOC);
1460                         if (IS_ERR(em)) {
1461                                 if (!nolock && nocow)
1462                                         btrfs_end_write_no_snapshoting(root);
1463                                 if (nocow)
1464                                         btrfs_dec_nocow_writers(fs_info,
1465                                                                 disk_bytenr);
1466                                 ret = PTR_ERR(em);
1467                                 goto error;
1468                         }
1469                         free_extent_map(em);
1470                 }
1471
1472                 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1473                         type = BTRFS_ORDERED_PREALLOC;
1474                 } else {
1475                         type = BTRFS_ORDERED_NOCOW;
1476                 }
1477
1478                 ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr,
1479                                                num_bytes, num_bytes, type);
1480                 if (nocow)
1481                         btrfs_dec_nocow_writers(fs_info, disk_bytenr);
1482                 BUG_ON(ret); /* -ENOMEM */
1483
1484                 if (root->root_key.objectid ==
1485                     BTRFS_DATA_RELOC_TREE_OBJECTID)
1486                         /*
1487                          * Error handled later, as we must prevent
1488                          * extent_clear_unlock_delalloc() in error handler
1489                          * from freeing metadata of created ordered extent.
1490                          */
1491                         ret = btrfs_reloc_clone_csums(inode, cur_offset,
1492                                                       num_bytes);
1493
1494                 extent_clear_unlock_delalloc(inode, cur_offset,
1495                                              cur_offset + num_bytes - 1, end,
1496                                              locked_page, EXTENT_LOCKED |
1497                                              EXTENT_DELALLOC |
1498                                              EXTENT_CLEAR_DATA_RESV,
1499                                              PAGE_UNLOCK | PAGE_SET_PRIVATE2);
1500
1501                 if (!nolock && nocow)
1502                         btrfs_end_write_no_snapshoting(root);
1503                 cur_offset = extent_end;
1504
1505                 /*
1506                  * btrfs_reloc_clone_csums() error, now we're OK to call error
1507                  * handler, as metadata for created ordered extent will only
1508                  * be freed by btrfs_finish_ordered_io().
1509                  */
1510                 if (ret)
1511                         goto error;
1512                 if (cur_offset > end)
1513                         break;
1514         }
1515         btrfs_release_path(path);
1516
1517         if (cur_offset <= end && cow_start == (u64)-1) {
1518                 cow_start = cur_offset;
1519                 cur_offset = end;
1520         }
1521
1522         if (cow_start != (u64)-1) {
1523                 ret = cow_file_range(inode, locked_page, cow_start, end, end,
1524                                      page_started, nr_written, 1, NULL);
1525                 if (ret)
1526                         goto error;
1527         }
1528
1529 error:
1530         if (ret && cur_offset < end)
1531                 extent_clear_unlock_delalloc(inode, cur_offset, end, end,
1532                                              locked_page, EXTENT_LOCKED |
1533                                              EXTENT_DELALLOC | EXTENT_DEFRAG |
1534                                              EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
1535                                              PAGE_CLEAR_DIRTY |
1536                                              PAGE_SET_WRITEBACK |
1537                                              PAGE_END_WRITEBACK);
1538         btrfs_free_path(path);
1539         return ret;
1540 }
1541
1542 static inline int need_force_cow(struct inode *inode, u64 start, u64 end)
1543 {
1544
1545         if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
1546             !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC))
1547                 return 0;
1548
1549         /*
1550          * @defrag_bytes is a hint value, no spinlock held here,
1551          * if is not zero, it means the file is defragging.
1552          * Force cow if given extent needs to be defragged.
1553          */
1554         if (BTRFS_I(inode)->defrag_bytes &&
1555             test_range_bit(&BTRFS_I(inode)->io_tree, start, end,
1556                            EXTENT_DEFRAG, 0, NULL))
1557                 return 1;
1558
1559         return 0;
1560 }
1561
1562 /*
1563  * extent_io.c call back to do delayed allocation processing
1564  */
1565 static int run_delalloc_range(void *private_data, struct page *locked_page,
1566                               u64 start, u64 end, int *page_started,
1567                               unsigned long *nr_written)
1568 {
1569         struct inode *inode = private_data;
1570         int ret;
1571         int force_cow = need_force_cow(inode, start, end);
1572
1573         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) {
1574                 ret = run_delalloc_nocow(inode, locked_page, start, end,
1575                                          page_started, 1, nr_written);
1576         } else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) {
1577                 ret = run_delalloc_nocow(inode, locked_page, start, end,
1578                                          page_started, 0, nr_written);
1579         } else if (!inode_need_compress(inode)) {
1580                 ret = cow_file_range(inode, locked_page, start, end, end,
1581                                       page_started, nr_written, 1, NULL);
1582         } else {
1583                 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
1584                         &BTRFS_I(inode)->runtime_flags);
1585                 ret = cow_file_range_async(inode, locked_page, start, end,
1586                                            page_started, nr_written);
1587         }
1588         if (ret)
1589                 btrfs_cleanup_ordered_extents(inode, start, end - start + 1);
1590         return ret;
1591 }
1592
1593 static void btrfs_split_extent_hook(void *private_data,
1594                                     struct extent_state *orig, u64 split)
1595 {
1596         struct inode *inode = private_data;
1597         u64 size;
1598
1599         /* not delalloc, ignore it */
1600         if (!(orig->state & EXTENT_DELALLOC))
1601                 return;
1602
1603         size = orig->end - orig->start + 1;
1604         if (size > BTRFS_MAX_EXTENT_SIZE) {
1605                 u32 num_extents;
1606                 u64 new_size;
1607
1608                 /*
1609                  * See the explanation in btrfs_merge_extent_hook, the same
1610                  * applies here, just in reverse.
1611                  */
1612                 new_size = orig->end - split + 1;
1613                 num_extents = count_max_extents(new_size);
1614                 new_size = split - orig->start;
1615                 num_extents += count_max_extents(new_size);
1616                 if (count_max_extents(size) >= num_extents)
1617                         return;
1618         }
1619
1620         spin_lock(&BTRFS_I(inode)->lock);
1621         BTRFS_I(inode)->outstanding_extents++;
1622         spin_unlock(&BTRFS_I(inode)->lock);
1623 }
1624
1625 /*
1626  * extent_io.c merge_extent_hook, used to track merged delayed allocation
1627  * extents so we can keep track of new extents that are just merged onto old
1628  * extents, such as when we are doing sequential writes, so we can properly
1629  * account for the metadata space we'll need.
1630  */
1631 static void btrfs_merge_extent_hook(void *private_data,
1632                                     struct extent_state *new,
1633                                     struct extent_state *other)
1634 {
1635         struct inode *inode = private_data;
1636         u64 new_size, old_size;
1637         u32 num_extents;
1638
1639         /* not delalloc, ignore it */
1640         if (!(other->state & EXTENT_DELALLOC))
1641                 return;
1642
1643         if (new->start > other->start)
1644                 new_size = new->end - other->start + 1;
1645         else
1646                 new_size = other->end - new->start + 1;
1647
1648         /* we're not bigger than the max, unreserve the space and go */
1649         if (new_size <= BTRFS_MAX_EXTENT_SIZE) {
1650                 spin_lock(&BTRFS_I(inode)->lock);
1651                 BTRFS_I(inode)->outstanding_extents--;
1652                 spin_unlock(&BTRFS_I(inode)->lock);
1653                 return;
1654         }
1655
1656         /*
1657          * We have to add up either side to figure out how many extents were
1658          * accounted for before we merged into one big extent.  If the number of
1659          * extents we accounted for is <= the amount we need for the new range
1660          * then we can return, otherwise drop.  Think of it like this
1661          *
1662          * [ 4k][MAX_SIZE]
1663          *
1664          * So we've grown the extent by a MAX_SIZE extent, this would mean we
1665          * need 2 outstanding extents, on one side we have 1 and the other side
1666          * we have 1 so they are == and we can return.  But in this case
1667          *
1668          * [MAX_SIZE+4k][MAX_SIZE+4k]
1669          *
1670          * Each range on their own accounts for 2 extents, but merged together
1671          * they are only 3 extents worth of accounting, so we need to drop in
1672          * this case.
1673          */
1674         old_size = other->end - other->start + 1;
1675         num_extents = count_max_extents(old_size);
1676         old_size = new->end - new->start + 1;
1677         num_extents += count_max_extents(old_size);
1678         if (count_max_extents(new_size) >= num_extents)
1679                 return;
1680
1681         spin_lock(&BTRFS_I(inode)->lock);
1682         BTRFS_I(inode)->outstanding_extents--;
1683         spin_unlock(&BTRFS_I(inode)->lock);
1684 }
1685
1686 static void btrfs_add_delalloc_inodes(struct btrfs_root *root,
1687                                       struct inode *inode)
1688 {
1689         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1690
1691         spin_lock(&root->delalloc_lock);
1692         if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
1693                 list_add_tail(&BTRFS_I(inode)->delalloc_inodes,
1694                               &root->delalloc_inodes);
1695                 set_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1696                         &BTRFS_I(inode)->runtime_flags);
1697                 root->nr_delalloc_inodes++;
1698                 if (root->nr_delalloc_inodes == 1) {
1699                         spin_lock(&fs_info->delalloc_root_lock);
1700                         BUG_ON(!list_empty(&root->delalloc_root));
1701                         list_add_tail(&root->delalloc_root,
1702                                       &fs_info->delalloc_roots);
1703                         spin_unlock(&fs_info->delalloc_root_lock);
1704                 }
1705         }
1706         spin_unlock(&root->delalloc_lock);
1707 }
1708
1709 static void btrfs_del_delalloc_inode(struct btrfs_root *root,
1710                                      struct btrfs_inode *inode)
1711 {
1712         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
1713
1714         spin_lock(&root->delalloc_lock);
1715         if (!list_empty(&inode->delalloc_inodes)) {
1716                 list_del_init(&inode->delalloc_inodes);
1717                 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1718                           &inode->runtime_flags);
1719                 root->nr_delalloc_inodes--;
1720                 if (!root->nr_delalloc_inodes) {
1721                         spin_lock(&fs_info->delalloc_root_lock);
1722                         BUG_ON(list_empty(&root->delalloc_root));
1723                         list_del_init(&root->delalloc_root);
1724                         spin_unlock(&fs_info->delalloc_root_lock);
1725                 }
1726         }
1727         spin_unlock(&root->delalloc_lock);
1728 }
1729
1730 /*
1731  * extent_io.c set_bit_hook, used to track delayed allocation
1732  * bytes in this file, and to maintain the list of inodes that
1733  * have pending delalloc work to be done.
1734  */
1735 static void btrfs_set_bit_hook(void *private_data,
1736                                struct extent_state *state, unsigned *bits)
1737 {
1738         struct inode *inode = private_data;
1739
1740         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1741
1742         if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC))
1743                 WARN_ON(1);
1744         /*
1745          * set_bit and clear bit hooks normally require _irqsave/restore
1746          * but in this case, we are only testing for the DELALLOC
1747          * bit, which is only set or cleared with irqs on
1748          */
1749         if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1750                 struct btrfs_root *root = BTRFS_I(inode)->root;
1751                 u64 len = state->end + 1 - state->start;
1752                 bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode));
1753
1754                 if (*bits & EXTENT_FIRST_DELALLOC) {
1755                         *bits &= ~EXTENT_FIRST_DELALLOC;
1756                 } else {
1757                         spin_lock(&BTRFS_I(inode)->lock);
1758                         BTRFS_I(inode)->outstanding_extents++;
1759                         spin_unlock(&BTRFS_I(inode)->lock);
1760                 }
1761
1762                 /* For sanity tests */
1763                 if (btrfs_is_testing(fs_info))
1764                         return;
1765
1766                 percpu_counter_add_batch(&fs_info->delalloc_bytes, len,
1767                                          fs_info->delalloc_batch);
1768                 spin_lock(&BTRFS_I(inode)->lock);
1769                 BTRFS_I(inode)->delalloc_bytes += len;
1770                 if (*bits & EXTENT_DEFRAG)
1771                         BTRFS_I(inode)->defrag_bytes += len;
1772                 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1773                                          &BTRFS_I(inode)->runtime_flags))
1774                         btrfs_add_delalloc_inodes(root, inode);
1775                 spin_unlock(&BTRFS_I(inode)->lock);
1776         }
1777
1778         if (!(state->state & EXTENT_DELALLOC_NEW) &&
1779             (*bits & EXTENT_DELALLOC_NEW)) {
1780                 spin_lock(&BTRFS_I(inode)->lock);
1781                 BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 -
1782                         state->start;
1783                 spin_unlock(&BTRFS_I(inode)->lock);
1784         }
1785 }
1786
1787 /*
1788  * extent_io.c clear_bit_hook, see set_bit_hook for why
1789  */
1790 static void btrfs_clear_bit_hook(void *private_data,
1791                                  struct extent_state *state,
1792                                  unsigned *bits)
1793 {
1794         struct btrfs_inode *inode = BTRFS_I((struct inode *)private_data);
1795         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
1796         u64 len = state->end + 1 - state->start;
1797         u32 num_extents = count_max_extents(len);
1798
1799         spin_lock(&inode->lock);
1800         if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG))
1801                 inode->defrag_bytes -= len;
1802         spin_unlock(&inode->lock);
1803
1804         /*
1805          * set_bit and clear bit hooks normally require _irqsave/restore
1806          * but in this case, we are only testing for the DELALLOC
1807          * bit, which is only set or cleared with irqs on
1808          */
1809         if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1810                 struct btrfs_root *root = inode->root;
1811                 bool do_list = !btrfs_is_free_space_inode(inode);
1812
1813                 if (*bits & EXTENT_FIRST_DELALLOC) {
1814                         *bits &= ~EXTENT_FIRST_DELALLOC;
1815                 } else if (!(*bits & EXTENT_CLEAR_META_RESV)) {
1816                         spin_lock(&inode->lock);
1817                         inode->outstanding_extents -= num_extents;
1818                         spin_unlock(&inode->lock);
1819                 }
1820
1821                 /*
1822                  * We don't reserve metadata space for space cache inodes so we
1823                  * don't need to call dellalloc_release_metadata if there is an
1824                  * error.
1825                  */
1826                 if (*bits & EXTENT_CLEAR_META_RESV &&
1827                     root != fs_info->tree_root)
1828                         btrfs_delalloc_release_metadata(inode, len);
1829
1830                 /* For sanity tests. */
1831                 if (btrfs_is_testing(fs_info))
1832                         return;
1833
1834                 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID &&
1835                     do_list && !(state->state & EXTENT_NORESERVE) &&
1836                     (*bits & EXTENT_CLEAR_DATA_RESV))
1837                         btrfs_free_reserved_data_space_noquota(
1838                                         &inode->vfs_inode,
1839                                         state->start, len);
1840
1841                 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len,
1842                                          fs_info->delalloc_batch);
1843                 spin_lock(&inode->lock);
1844                 inode->delalloc_bytes -= len;
1845                 if (do_list && inode->delalloc_bytes == 0 &&
1846                     test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1847                                         &inode->runtime_flags))
1848                         btrfs_del_delalloc_inode(root, inode);
1849                 spin_unlock(&inode->lock);
1850         }
1851
1852         if ((state->state & EXTENT_DELALLOC_NEW) &&
1853             (*bits & EXTENT_DELALLOC_NEW)) {
1854                 spin_lock(&inode->lock);
1855                 ASSERT(inode->new_delalloc_bytes >= len);
1856                 inode->new_delalloc_bytes -= len;
1857                 spin_unlock(&inode->lock);
1858         }
1859 }
1860
1861 /*
1862  * extent_io.c merge_bio_hook, this must check the chunk tree to make sure
1863  * we don't create bios that span stripes or chunks
1864  *
1865  * return 1 if page cannot be merged to bio
1866  * return 0 if page can be merged to bio
1867  * return error otherwise
1868  */
1869 int btrfs_merge_bio_hook(struct page *page, unsigned long offset,
1870                          size_t size, struct bio *bio,
1871                          unsigned long bio_flags)
1872 {
1873         struct inode *inode = page->mapping->host;
1874         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1875         u64 logical = (u64)bio->bi_iter.bi_sector << 9;
1876         u64 length = 0;
1877         u64 map_length;
1878         int ret;
1879
1880         if (bio_flags & EXTENT_BIO_COMPRESSED)
1881                 return 0;
1882
1883         length = bio->bi_iter.bi_size;
1884         map_length = length;
1885         ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length,
1886                               NULL, 0);
1887         if (ret < 0)
1888                 return ret;
1889         if (map_length < length + size)
1890                 return 1;
1891         return 0;
1892 }
1893
1894 /*
1895  * in order to insert checksums into the metadata in large chunks,
1896  * we wait until bio submission time.   All the pages in the bio are
1897  * checksummed and sums are attached onto the ordered extent record.
1898  *
1899  * At IO completion time the cums attached on the ordered extent record
1900  * are inserted into the btree
1901  */
1902 static blk_status_t __btrfs_submit_bio_start(void *private_data, struct bio *bio,
1903                                     int mirror_num, unsigned long bio_flags,
1904                                     u64 bio_offset)
1905 {
1906         struct inode *inode = private_data;
1907         blk_status_t ret = 0;
1908
1909         ret = btrfs_csum_one_bio(inode, bio, 0, 0);
1910         BUG_ON(ret); /* -ENOMEM */
1911         return 0;
1912 }
1913
1914 /*
1915  * in order to insert checksums into the metadata in large chunks,
1916  * we wait until bio submission time.   All the pages in the bio are
1917  * checksummed and sums are attached onto the ordered extent record.
1918  *
1919  * At IO completion time the cums attached on the ordered extent record
1920  * are inserted into the btree
1921  */
1922 static blk_status_t __btrfs_submit_bio_done(void *private_data, struct bio *bio,
1923                           int mirror_num, unsigned long bio_flags,
1924                           u64 bio_offset)
1925 {
1926         struct inode *inode = private_data;
1927         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1928         blk_status_t ret;
1929
1930         ret = btrfs_map_bio(fs_info, bio, mirror_num, 1);
1931         if (ret) {
1932                 bio->bi_status = ret;
1933                 bio_endio(bio);
1934         }
1935         return ret;
1936 }
1937
1938 /*
1939  * extent_io.c submission hook. This does the right thing for csum calculation
1940  * on write, or reading the csums from the tree before a read
1941  */
1942 static blk_status_t btrfs_submit_bio_hook(void *private_data, struct bio *bio,
1943                                  int mirror_num, unsigned long bio_flags,
1944                                  u64 bio_offset)
1945 {
1946         struct inode *inode = private_data;
1947         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1948         struct btrfs_root *root = BTRFS_I(inode)->root;
1949         enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA;
1950         blk_status_t ret = 0;
1951         int skip_sum;
1952         int async = !atomic_read(&BTRFS_I(inode)->sync_writers);
1953
1954         skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
1955
1956         if (btrfs_is_free_space_inode(BTRFS_I(inode)))
1957                 metadata = BTRFS_WQ_ENDIO_FREE_SPACE;
1958
1959         if (bio_op(bio) != REQ_OP_WRITE) {
1960                 ret = btrfs_bio_wq_end_io(fs_info, bio, metadata);
1961                 if (ret)
1962                         goto out;
1963
1964                 if (bio_flags & EXTENT_BIO_COMPRESSED) {
1965                         ret = btrfs_submit_compressed_read(inode, bio,
1966                                                            mirror_num,
1967                                                            bio_flags);
1968                         goto out;
1969                 } else if (!skip_sum) {
1970                         ret = btrfs_lookup_bio_sums(inode, bio, NULL);
1971                         if (ret)
1972                                 goto out;
1973                 }
1974                 goto mapit;
1975         } else if (async && !skip_sum) {
1976                 /* csum items have already been cloned */
1977                 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
1978                         goto mapit;
1979                 /* we're doing a write, do the async checksumming */
1980                 ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, bio_flags,
1981                                           bio_offset, inode,
1982                                           __btrfs_submit_bio_start,
1983                                           __btrfs_submit_bio_done);
1984                 goto out;
1985         } else if (!skip_sum) {
1986                 ret = btrfs_csum_one_bio(inode, bio, 0, 0);
1987                 if (ret)
1988                         goto out;
1989         }
1990
1991 mapit:
1992         ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
1993
1994 out:
1995         if (ret) {
1996                 bio->bi_status = ret;
1997                 bio_endio(bio);
1998         }
1999         return ret;
2000 }
2001
2002 /*
2003  * given a list of ordered sums record them in the inode.  This happens
2004  * at IO completion time based on sums calculated at bio submission time.
2005  */
2006 static noinline int add_pending_csums(struct btrfs_trans_handle *trans,
2007                              struct inode *inode, struct list_head *list)
2008 {
2009         struct btrfs_ordered_sum *sum;
2010
2011         list_for_each_entry(sum, list, list) {
2012                 trans->adding_csums = 1;
2013                 btrfs_csum_file_blocks(trans,
2014                        BTRFS_I(inode)->root->fs_info->csum_root, sum);
2015                 trans->adding_csums = 0;
2016         }
2017         return 0;
2018 }
2019
2020 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end,
2021                               struct extent_state **cached_state, int dedupe)
2022 {
2023         WARN_ON((end & (PAGE_SIZE - 1)) == 0);
2024         return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end,
2025                                    cached_state);
2026 }
2027
2028 /* see btrfs_writepage_start_hook for details on why this is required */
2029 struct btrfs_writepage_fixup {
2030         struct page *page;
2031         struct btrfs_work work;
2032 };
2033
2034 static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
2035 {
2036         struct btrfs_writepage_fixup *fixup;
2037         struct btrfs_ordered_extent *ordered;
2038         struct extent_state *cached_state = NULL;
2039         struct extent_changeset *data_reserved = NULL;
2040         struct page *page;
2041         struct inode *inode;
2042         u64 page_start;
2043         u64 page_end;
2044         int ret;
2045
2046         fixup = container_of(work, struct btrfs_writepage_fixup, work);
2047         page = fixup->page;
2048 again:
2049         lock_page(page);
2050         if (!page->mapping || !PageDirty(page) || !PageChecked(page)) {
2051                 ClearPageChecked(page);
2052                 goto out_page;
2053         }
2054
2055         inode = page->mapping->host;
2056         page_start = page_offset(page);
2057         page_end = page_offset(page) + PAGE_SIZE - 1;
2058
2059         lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end,
2060                          &cached_state);
2061
2062         /* already ordered? We're done */
2063         if (PagePrivate2(page))
2064                 goto out;
2065
2066         ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
2067                                         PAGE_SIZE);
2068         if (ordered) {
2069                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start,
2070                                      page_end, &cached_state, GFP_NOFS);
2071                 unlock_page(page);
2072                 btrfs_start_ordered_extent(inode, ordered, 1);
2073                 btrfs_put_ordered_extent(ordered);
2074                 goto again;
2075         }
2076
2077         ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
2078                                            PAGE_SIZE);
2079         if (ret) {
2080                 mapping_set_error(page->mapping, ret);
2081                 end_extent_writepage(page, ret, page_start, page_end);
2082                 ClearPageChecked(page);
2083                 goto out;
2084          }
2085
2086         btrfs_set_extent_delalloc(inode, page_start, page_end, &cached_state,
2087                                   0);
2088         ClearPageChecked(page);
2089         set_page_dirty(page);
2090 out:
2091         unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end,
2092                              &cached_state, GFP_NOFS);
2093 out_page:
2094         unlock_page(page);
2095         put_page(page);
2096         kfree(fixup);
2097         extent_changeset_free(data_reserved);
2098 }
2099
2100 /*
2101  * There are a few paths in the higher layers of the kernel that directly
2102  * set the page dirty bit without asking the filesystem if it is a
2103  * good idea.  This causes problems because we want to make sure COW
2104  * properly happens and the data=ordered rules are followed.
2105  *
2106  * In our case any range that doesn't have the ORDERED bit set
2107  * hasn't been properly setup for IO.  We kick off an async process
2108  * to fix it up.  The async helper will wait for ordered extents, set
2109  * the delalloc bit and make it safe to write the page.
2110  */
2111 static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end)
2112 {
2113         struct inode *inode = page->mapping->host;
2114         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2115         struct btrfs_writepage_fixup *fixup;
2116
2117         /* this page is properly in the ordered list */
2118         if (TestClearPagePrivate2(page))
2119                 return 0;
2120
2121         if (PageChecked(page))
2122                 return -EAGAIN;
2123
2124         fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
2125         if (!fixup)
2126                 return -EAGAIN;
2127
2128         SetPageChecked(page);
2129         get_page(page);
2130         btrfs_init_work(&fixup->work, btrfs_fixup_helper,
2131                         btrfs_writepage_fixup_worker, NULL, NULL);
2132         fixup->page = page;
2133         btrfs_queue_work(fs_info->fixup_workers, &fixup->work);
2134         return -EBUSY;
2135 }
2136
2137 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
2138                                        struct inode *inode, u64 file_pos,
2139                                        u64 disk_bytenr, u64 disk_num_bytes,
2140                                        u64 num_bytes, u64 ram_bytes,
2141                                        u8 compression, u8 encryption,
2142                                        u16 other_encoding, int extent_type)
2143 {
2144         struct btrfs_root *root = BTRFS_I(inode)->root;
2145         struct btrfs_file_extent_item *fi;
2146         struct btrfs_path *path;
2147         struct extent_buffer *leaf;
2148         struct btrfs_key ins;
2149         u64 qg_released;
2150         int extent_inserted = 0;
2151         int ret;
2152
2153         path = btrfs_alloc_path();
2154         if (!path)
2155                 return -ENOMEM;
2156
2157         /*
2158          * we may be replacing one extent in the tree with another.
2159          * The new extent is pinned in the extent map, and we don't want
2160          * to drop it from the cache until it is completely in the btree.
2161          *
2162          * So, tell btrfs_drop_extents to leave this extent in the cache.
2163          * the caller is expected to unpin it and allow it to be merged
2164          * with the others.
2165          */
2166         ret = __btrfs_drop_extents(trans, root, inode, path, file_pos,
2167                                    file_pos + num_bytes, NULL, 0,
2168                                    1, sizeof(*fi), &extent_inserted);
2169         if (ret)
2170                 goto out;
2171
2172         if (!extent_inserted) {
2173                 ins.objectid = btrfs_ino(BTRFS_I(inode));
2174                 ins.offset = file_pos;
2175                 ins.type = BTRFS_EXTENT_DATA_KEY;
2176
2177                 path->leave_spinning = 1;
2178                 ret = btrfs_insert_empty_item(trans, root, path, &ins,
2179                                               sizeof(*fi));
2180                 if (ret)
2181                         goto out;
2182         }
2183         leaf = path->nodes[0];
2184         fi = btrfs_item_ptr(leaf, path->slots[0],
2185                             struct btrfs_file_extent_item);
2186         btrfs_set_file_extent_generation(leaf, fi, trans->transid);
2187         btrfs_set_file_extent_type(leaf, fi, extent_type);
2188         btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr);
2189         btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes);
2190         btrfs_set_file_extent_offset(leaf, fi, 0);
2191         btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2192         btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes);
2193         btrfs_set_file_extent_compression(leaf, fi, compression);
2194         btrfs_set_file_extent_encryption(leaf, fi, encryption);
2195         btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding);
2196
2197         btrfs_mark_buffer_dirty(leaf);
2198         btrfs_release_path(path);
2199
2200         inode_add_bytes(inode, num_bytes);
2201
2202         ins.objectid = disk_bytenr;
2203         ins.offset = disk_num_bytes;
2204         ins.type = BTRFS_EXTENT_ITEM_KEY;
2205
2206         /*
2207          * Release the reserved range from inode dirty range map, as it is
2208          * already moved into delayed_ref_head
2209          */
2210         ret = btrfs_qgroup_release_data(inode, file_pos, ram_bytes);
2211         if (ret < 0)
2212                 goto out;
2213         qg_released = ret;
2214         ret = btrfs_alloc_reserved_file_extent(trans, root->root_key.objectid,
2215                         btrfs_ino(BTRFS_I(inode)), file_pos, qg_released, &ins);
2216 out:
2217         btrfs_free_path(path);
2218
2219         return ret;
2220 }
2221
2222 /* snapshot-aware defrag */
2223 struct sa_defrag_extent_backref {
2224         struct rb_node node;
2225         struct old_sa_defrag_extent *old;
2226         u64 root_id;
2227         u64 inum;
2228         u64 file_pos;
2229         u64 extent_offset;
2230         u64 num_bytes;
2231         u64 generation;
2232 };
2233
2234 struct old_sa_defrag_extent {
2235         struct list_head list;
2236         struct new_sa_defrag_extent *new;
2237
2238         u64 extent_offset;
2239         u64 bytenr;
2240         u64 offset;
2241         u64 len;
2242         int count;
2243 };
2244
2245 struct new_sa_defrag_extent {
2246         struct rb_root root;
2247         struct list_head head;
2248         struct btrfs_path *path;
2249         struct inode *inode;
2250         u64 file_pos;
2251         u64 len;
2252         u64 bytenr;
2253         u64 disk_len;
2254         u8 compress_type;
2255 };
2256
2257 static int backref_comp(struct sa_defrag_extent_backref *b1,
2258                         struct sa_defrag_extent_backref *b2)
2259 {
2260         if (b1->root_id < b2->root_id)
2261                 return -1;
2262         else if (b1->root_id > b2->root_id)
2263                 return 1;
2264
2265         if (b1->inum < b2->inum)
2266                 return -1;
2267         else if (b1->inum > b2->inum)
2268                 return 1;
2269
2270         if (b1->file_pos < b2->file_pos)
2271                 return -1;
2272         else if (b1->file_pos > b2->file_pos)
2273                 return 1;
2274
2275         /*
2276          * [------------------------------] ===> (a range of space)
2277          *     |<--->|   |<---->| =============> (fs/file tree A)
2278          * |<---------------------------->| ===> (fs/file tree B)
2279          *
2280          * A range of space can refer to two file extents in one tree while
2281          * refer to only one file extent in another tree.
2282          *
2283          * So we may process a disk offset more than one time(two extents in A)
2284          * and locate at the same extent(one extent in B), then insert two same
2285          * backrefs(both refer to the extent in B).
2286          */
2287         return 0;
2288 }
2289
2290 static void backref_insert(struct rb_root *root,
2291                            struct sa_defrag_extent_backref *backref)
2292 {
2293         struct rb_node **p = &root->rb_node;
2294         struct rb_node *parent = NULL;
2295         struct sa_defrag_extent_backref *entry;
2296         int ret;
2297
2298         while (*p) {
2299                 parent = *p;
2300                 entry = rb_entry(parent, struct sa_defrag_extent_backref, node);
2301
2302                 ret = backref_comp(backref, entry);
2303                 if (ret < 0)
2304                         p = &(*p)->rb_left;
2305                 else
2306                         p = &(*p)->rb_right;
2307         }
2308
2309         rb_link_node(&backref->node, parent, p);
2310         rb_insert_color(&backref->node, root);
2311 }
2312
2313 /*
2314  * Note the backref might has changed, and in this case we just return 0.
2315  */
2316 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id,
2317                                        void *ctx)
2318 {
2319         struct btrfs_file_extent_item *extent;
2320         struct old_sa_defrag_extent *old = ctx;
2321         struct new_sa_defrag_extent *new = old->new;
2322         struct btrfs_path *path = new->path;
2323         struct btrfs_key key;
2324         struct btrfs_root *root;
2325         struct sa_defrag_extent_backref *backref;
2326         struct extent_buffer *leaf;
2327         struct inode *inode = new->inode;
2328         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2329         int slot;
2330         int ret;
2331         u64 extent_offset;
2332         u64 num_bytes;
2333
2334         if (BTRFS_I(inode)->root->root_key.objectid == root_id &&
2335             inum == btrfs_ino(BTRFS_I(inode)))
2336                 return 0;
2337
2338         key.objectid = root_id;
2339         key.type = BTRFS_ROOT_ITEM_KEY;
2340         key.offset = (u64)-1;
2341
2342         root = btrfs_read_fs_root_no_name(fs_info, &key);
2343         if (IS_ERR(root)) {
2344                 if (PTR_ERR(root) == -ENOENT)
2345                         return 0;
2346                 WARN_ON(1);
2347                 btrfs_debug(fs_info, "inum=%llu, offset=%llu, root_id=%llu",
2348                          inum, offset, root_id);
2349                 return PTR_ERR(root);
2350         }
2351
2352         key.objectid = inum;
2353         key.type = BTRFS_EXTENT_DATA_KEY;
2354         if (offset > (u64)-1 << 32)
2355                 key.offset = 0;
2356         else
2357                 key.offset = offset;
2358
2359         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2360         if (WARN_ON(ret < 0))
2361                 return ret;
2362         ret = 0;
2363
2364         while (1) {
2365                 cond_resched();
2366
2367                 leaf = path->nodes[0];
2368                 slot = path->slots[0];
2369
2370                 if (slot >= btrfs_header_nritems(leaf)) {
2371                         ret = btrfs_next_leaf(root, path);
2372                         if (ret < 0) {
2373                                 goto out;
2374                         } else if (ret > 0) {
2375                                 ret = 0;
2376                                 goto out;
2377                         }
2378                         continue;
2379                 }
2380
2381                 path->slots[0]++;
2382
2383                 btrfs_item_key_to_cpu(leaf, &key, slot);
2384
2385                 if (key.objectid > inum)
2386                         goto out;
2387
2388                 if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY)
2389                         continue;
2390
2391                 extent = btrfs_item_ptr(leaf, slot,
2392                                         struct btrfs_file_extent_item);
2393
2394                 if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr)
2395                         continue;
2396
2397                 /*
2398                  * 'offset' refers to the exact key.offset,
2399                  * NOT the 'offset' field in btrfs_extent_data_ref, ie.
2400                  * (key.offset - extent_offset).
2401                  */
2402                 if (key.offset != offset)
2403                         continue;
2404
2405                 extent_offset = btrfs_file_extent_offset(leaf, extent);
2406                 num_bytes = btrfs_file_extent_num_bytes(leaf, extent);
2407
2408                 if (extent_offset >= old->extent_offset + old->offset +
2409                     old->len || extent_offset + num_bytes <=
2410                     old->extent_offset + old->offset)
2411                         continue;
2412                 break;
2413         }
2414
2415         backref = kmalloc(sizeof(*backref), GFP_NOFS);
2416         if (!backref) {
2417                 ret = -ENOENT;
2418                 goto out;
2419         }
2420
2421         backref->root_id = root_id;
2422         backref->inum = inum;
2423         backref->file_pos = offset;
2424         backref->num_bytes = num_bytes;
2425         backref->extent_offset = extent_offset;
2426         backref->generation = btrfs_file_extent_generation(leaf, extent);
2427         backref->old = old;
2428         backref_insert(&new->root, backref);
2429         old->count++;
2430 out:
2431         btrfs_release_path(path);
2432         WARN_ON(ret);
2433         return ret;
2434 }
2435
2436 static noinline bool record_extent_backrefs(struct btrfs_path *path,
2437                                    struct new_sa_defrag_extent *new)
2438 {
2439         struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2440         struct old_sa_defrag_extent *old, *tmp;
2441         int ret;
2442
2443         new->path = path;
2444
2445         list_for_each_entry_safe(old, tmp, &new->head, list) {
2446                 ret = iterate_inodes_from_logical(old->bytenr +
2447                                                   old->extent_offset, fs_info,
2448                                                   path, record_one_backref,
2449                                                   old);
2450                 if (ret < 0 && ret != -ENOENT)
2451                         return false;
2452
2453                 /* no backref to be processed for this extent */
2454                 if (!old->count) {
2455                         list_del(&old->list);
2456                         kfree(old);
2457                 }
2458         }
2459
2460         if (list_empty(&new->head))
2461                 return false;
2462
2463         return true;
2464 }
2465
2466 static int relink_is_mergable(struct extent_buffer *leaf,
2467                               struct btrfs_file_extent_item *fi,
2468                               struct new_sa_defrag_extent *new)
2469 {
2470         if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr)
2471                 return 0;
2472
2473         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2474                 return 0;
2475
2476         if (btrfs_file_extent_compression(leaf, fi) != new->compress_type)
2477                 return 0;
2478
2479         if (btrfs_file_extent_encryption(leaf, fi) ||
2480             btrfs_file_extent_other_encoding(leaf, fi))
2481                 return 0;
2482
2483         return 1;
2484 }
2485
2486 /*
2487  * Note the backref might has changed, and in this case we just return 0.
2488  */
2489 static noinline int relink_extent_backref(struct btrfs_path *path,
2490                                  struct sa_defrag_extent_backref *prev,
2491                                  struct sa_defrag_extent_backref *backref)
2492 {
2493         struct btrfs_file_extent_item *extent;
2494         struct btrfs_file_extent_item *item;
2495         struct btrfs_ordered_extent *ordered;
2496         struct btrfs_trans_handle *trans;
2497         struct btrfs_root *root;
2498         struct btrfs_key key;
2499         struct extent_buffer *leaf;
2500         struct old_sa_defrag_extent *old = backref->old;
2501         struct new_sa_defrag_extent *new = old->new;
2502         struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2503         struct inode *inode;
2504         struct extent_state *cached = NULL;
2505         int ret = 0;
2506         u64 start;
2507         u64 len;
2508         u64 lock_start;
2509         u64 lock_end;
2510         bool merge = false;
2511         int index;
2512
2513         if (prev && prev->root_id == backref->root_id &&
2514             prev->inum == backref->inum &&
2515             prev->file_pos + prev->num_bytes == backref->file_pos)
2516                 merge = true;
2517
2518         /* step 1: get root */
2519         key.objectid = backref->root_id;
2520         key.type = BTRFS_ROOT_ITEM_KEY;
2521         key.offset = (u64)-1;
2522
2523         index = srcu_read_lock(&fs_info->subvol_srcu);
2524
2525         root = btrfs_read_fs_root_no_name(fs_info, &key);
2526         if (IS_ERR(root)) {
2527                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2528                 if (PTR_ERR(root) == -ENOENT)
2529                         return 0;
2530                 return PTR_ERR(root);
2531         }
2532
2533         if (btrfs_root_readonly(root)) {
2534                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2535                 return 0;
2536         }
2537
2538         /* step 2: get inode */
2539         key.objectid = backref->inum;
2540         key.type = BTRFS_INODE_ITEM_KEY;
2541         key.offset = 0;
2542
2543         inode = btrfs_iget(fs_info->sb, &key, root, NULL);
2544         if (IS_ERR(inode)) {
2545                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2546                 return 0;
2547         }
2548
2549         srcu_read_unlock(&fs_info->subvol_srcu, index);
2550
2551         /* step 3: relink backref */
2552         lock_start = backref->file_pos;
2553         lock_end = backref->file_pos + backref->num_bytes - 1;
2554         lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2555                          &cached);
2556
2557         ordered = btrfs_lookup_first_ordered_extent(inode, lock_end);
2558         if (ordered) {
2559                 btrfs_put_ordered_extent(ordered);
2560                 goto out_unlock;
2561         }
2562
2563         trans = btrfs_join_transaction(root);
2564         if (IS_ERR(trans)) {
2565                 ret = PTR_ERR(trans);
2566                 goto out_unlock;
2567         }
2568
2569         key.objectid = backref->inum;
2570         key.type = BTRFS_EXTENT_DATA_KEY;
2571         key.offset = backref->file_pos;
2572
2573         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2574         if (ret < 0) {
2575                 goto out_free_path;
2576         } else if (ret > 0) {
2577                 ret = 0;
2578                 goto out_free_path;
2579         }
2580
2581         extent = btrfs_item_ptr(path->nodes[0], path->slots[0],
2582                                 struct btrfs_file_extent_item);
2583
2584         if (btrfs_file_extent_generation(path->nodes[0], extent) !=
2585             backref->generation)
2586                 goto out_free_path;
2587
2588         btrfs_release_path(path);
2589
2590         start = backref->file_pos;
2591         if (backref->extent_offset < old->extent_offset + old->offset)
2592                 start += old->extent_offset + old->offset -
2593                          backref->extent_offset;
2594
2595         len = min(backref->extent_offset + backref->num_bytes,
2596                   old->extent_offset + old->offset + old->len);
2597         len -= max(backref->extent_offset, old->extent_offset + old->offset);
2598
2599         ret = btrfs_drop_extents(trans, root, inode, start,
2600                                  start + len, 1);
2601         if (ret)
2602                 goto out_free_path;
2603 again:
2604         key.objectid = btrfs_ino(BTRFS_I(inode));
2605         key.type = BTRFS_EXTENT_DATA_KEY;
2606         key.offset = start;
2607
2608         path->leave_spinning = 1;
2609         if (merge) {
2610                 struct btrfs_file_extent_item *fi;
2611                 u64 extent_len;
2612                 struct btrfs_key found_key;
2613
2614                 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2615                 if (ret < 0)
2616                         goto out_free_path;
2617
2618                 path->slots[0]--;
2619                 leaf = path->nodes[0];
2620                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
2621
2622                 fi = btrfs_item_ptr(leaf, path->slots[0],
2623                                     struct btrfs_file_extent_item);
2624                 extent_len = btrfs_file_extent_num_bytes(leaf, fi);
2625
2626                 if (extent_len + found_key.offset == start &&
2627                     relink_is_mergable(leaf, fi, new)) {
2628                         btrfs_set_file_extent_num_bytes(leaf, fi,
2629                                                         extent_len + len);
2630                         btrfs_mark_buffer_dirty(leaf);
2631                         inode_add_bytes(inode, len);
2632
2633                         ret = 1;
2634                         goto out_free_path;
2635                 } else {
2636                         merge = false;
2637                         btrfs_release_path(path);
2638                         goto again;
2639                 }
2640         }
2641
2642         ret = btrfs_insert_empty_item(trans, root, path, &key,
2643                                         sizeof(*extent));
2644         if (ret) {
2645                 btrfs_abort_transaction(trans, ret);
2646                 goto out_free_path;
2647         }
2648
2649         leaf = path->nodes[0];
2650         item = btrfs_item_ptr(leaf, path->slots[0],
2651                                 struct btrfs_file_extent_item);
2652         btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr);
2653         btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len);
2654         btrfs_set_file_extent_offset(leaf, item, start - new->file_pos);
2655         btrfs_set_file_extent_num_bytes(leaf, item, len);
2656         btrfs_set_file_extent_ram_bytes(leaf, item, new->len);
2657         btrfs_set_file_extent_generation(leaf, item, trans->transid);
2658         btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
2659         btrfs_set_file_extent_compression(leaf, item, new->compress_type);
2660         btrfs_set_file_extent_encryption(leaf, item, 0);
2661         btrfs_set_file_extent_other_encoding(leaf, item, 0);
2662
2663         btrfs_mark_buffer_dirty(leaf);
2664         inode_add_bytes(inode, len);
2665         btrfs_release_path(path);
2666
2667         ret = btrfs_inc_extent_ref(trans, fs_info, new->bytenr,
2668                         new->disk_len, 0,
2669                         backref->root_id, backref->inum,
2670                         new->file_pos); /* start - extent_offset */
2671         if (ret) {
2672                 btrfs_abort_transaction(trans, ret);
2673                 goto out_free_path;
2674         }
2675
2676         ret = 1;
2677 out_free_path:
2678         btrfs_release_path(path);
2679         path->leave_spinning = 0;
2680         btrfs_end_transaction(trans);
2681 out_unlock:
2682         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2683                              &cached, GFP_NOFS);
2684         iput(inode);
2685         return ret;
2686 }
2687
2688 static void free_sa_defrag_extent(struct new_sa_defrag_extent *new)
2689 {
2690         struct old_sa_defrag_extent *old, *tmp;
2691
2692         if (!new)
2693                 return;
2694
2695         list_for_each_entry_safe(old, tmp, &new->head, list) {
2696                 kfree(old);
2697         }
2698         kfree(new);
2699 }
2700
2701 static void relink_file_extents(struct new_sa_defrag_extent *new)
2702 {
2703         struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2704         struct btrfs_path *path;
2705         struct sa_defrag_extent_backref *backref;
2706         struct sa_defrag_extent_backref *prev = NULL;
2707         struct inode *inode;
2708         struct btrfs_root *root;
2709         struct rb_node *node;
2710         int ret;
2711
2712         inode = new->inode;
2713         root = BTRFS_I(inode)->root;
2714
2715         path = btrfs_alloc_path();
2716         if (!path)
2717                 return;
2718
2719         if (!record_extent_backrefs(path, new)) {
2720                 btrfs_free_path(path);
2721                 goto out;
2722         }
2723         btrfs_release_path(path);
2724
2725         while (1) {
2726                 node = rb_first(&new->root);
2727                 if (!node)
2728                         break;
2729                 rb_erase(node, &new->root);
2730
2731                 backref = rb_entry(node, struct sa_defrag_extent_backref, node);
2732
2733                 ret = relink_extent_backref(path, prev, backref);
2734                 WARN_ON(ret < 0);
2735
2736                 kfree(prev);
2737
2738                 if (ret == 1)
2739                         prev = backref;
2740                 else
2741                         prev = NULL;
2742                 cond_resched();
2743         }
2744         kfree(prev);
2745
2746         btrfs_free_path(path);
2747 out:
2748         free_sa_defrag_extent(new);
2749
2750         atomic_dec(&fs_info->defrag_running);
2751         wake_up(&fs_info->transaction_wait);
2752 }
2753
2754 static struct new_sa_defrag_extent *
2755 record_old_file_extents(struct inode *inode,
2756                         struct btrfs_ordered_extent *ordered)
2757 {
2758         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2759         struct btrfs_root *root = BTRFS_I(inode)->root;
2760         struct btrfs_path *path;
2761         struct btrfs_key key;
2762         struct old_sa_defrag_extent *old;
2763         struct new_sa_defrag_extent *new;
2764         int ret;
2765
2766         new = kmalloc(sizeof(*new), GFP_NOFS);
2767         if (!new)
2768                 return NULL;
2769
2770         new->inode = inode;
2771         new->file_pos = ordered->file_offset;
2772         new->len = ordered->len;
2773         new->bytenr = ordered->start;
2774         new->disk_len = ordered->disk_len;
2775         new->compress_type = ordered->compress_type;
2776         new->root = RB_ROOT;
2777         INIT_LIST_HEAD(&new->head);
2778
2779         path = btrfs_alloc_path();
2780         if (!path)
2781                 goto out_kfree;
2782
2783         key.objectid = btrfs_ino(BTRFS_I(inode));
2784         key.type = BTRFS_EXTENT_DATA_KEY;
2785         key.offset = new->file_pos;
2786
2787         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2788         if (ret < 0)
2789                 goto out_free_path;
2790         if (ret > 0 && path->slots[0] > 0)
2791                 path->slots[0]--;
2792
2793         /* find out all the old extents for the file range */
2794         while (1) {
2795                 struct btrfs_file_extent_item *extent;
2796                 struct extent_buffer *l;
2797                 int slot;
2798                 u64 num_bytes;
2799                 u64 offset;
2800                 u64 end;
2801                 u64 disk_bytenr;
2802                 u64 extent_offset;
2803
2804                 l = path->nodes[0];
2805                 slot = path->slots[0];
2806
2807                 if (slot >= btrfs_header_nritems(l)) {
2808                         ret = btrfs_next_leaf(root, path);
2809                         if (ret < 0)
2810                                 goto out_free_path;
2811                         else if (ret > 0)
2812                                 break;
2813                         continue;
2814                 }
2815
2816                 btrfs_item_key_to_cpu(l, &key, slot);
2817
2818                 if (key.objectid != btrfs_ino(BTRFS_I(inode)))
2819                         break;
2820                 if (key.type != BTRFS_EXTENT_DATA_KEY)
2821                         break;
2822                 if (key.offset >= new->file_pos + new->len)
2823                         break;
2824
2825                 extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item);
2826
2827                 num_bytes = btrfs_file_extent_num_bytes(l, extent);
2828                 if (key.offset + num_bytes < new->file_pos)
2829                         goto next;
2830
2831                 disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent);
2832                 if (!disk_bytenr)
2833                         goto next;
2834
2835                 extent_offset = btrfs_file_extent_offset(l, extent);
2836
2837                 old = kmalloc(sizeof(*old), GFP_NOFS);
2838                 if (!old)
2839                         goto out_free_path;
2840
2841                 offset = max(new->file_pos, key.offset);
2842                 end = min(new->file_pos + new->len, key.offset + num_bytes);
2843
2844                 old->bytenr = disk_bytenr;
2845                 old->extent_offset = extent_offset;
2846                 old->offset = offset - key.offset;
2847                 old->len = end - offset;
2848                 old->new = new;
2849                 old->count = 0;
2850                 list_add_tail(&old->list, &new->head);
2851 next:
2852                 path->slots[0]++;
2853                 cond_resched();
2854         }
2855
2856         btrfs_free_path(path);
2857         atomic_inc(&fs_info->defrag_running);
2858
2859         return new;
2860
2861 out_free_path:
2862         btrfs_free_path(path);
2863 out_kfree:
2864         free_sa_defrag_extent(new);
2865         return NULL;
2866 }
2867
2868 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info,
2869                                          u64 start, u64 len)
2870 {
2871         struct btrfs_block_group_cache *cache;
2872
2873         cache = btrfs_lookup_block_group(fs_info, start);
2874         ASSERT(cache);
2875
2876         spin_lock(&cache->lock);
2877         cache->delalloc_bytes -= len;
2878         spin_unlock(&cache->lock);
2879
2880         btrfs_put_block_group(cache);
2881 }
2882
2883 /* as ordered data IO finishes, this gets called so we can finish
2884  * an ordered extent if the range of bytes in the file it covers are
2885  * fully written.
2886  */
2887 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
2888 {
2889         struct inode *inode = ordered_extent->inode;
2890         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2891         struct btrfs_root *root = BTRFS_I(inode)->root;
2892         struct btrfs_trans_handle *trans = NULL;
2893         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
2894         struct extent_state *cached_state = NULL;
2895         struct new_sa_defrag_extent *new = NULL;
2896         int compress_type = 0;
2897         int ret = 0;
2898         u64 logical_len = ordered_extent->len;
2899         bool nolock;
2900         bool truncated = false;
2901         bool range_locked = false;
2902         bool clear_new_delalloc_bytes = false;
2903
2904         if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
2905             !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) &&
2906             !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags))
2907                 clear_new_delalloc_bytes = true;
2908
2909         nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
2910
2911         if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
2912                 ret = -EIO;
2913                 goto out;
2914         }
2915
2916         btrfs_free_io_failure_record(BTRFS_I(inode),
2917                         ordered_extent->file_offset,
2918                         ordered_extent->file_offset +
2919                         ordered_extent->len - 1);
2920
2921         if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
2922                 truncated = true;
2923                 logical_len = ordered_extent->truncated_len;
2924                 /* Truncated the entire extent, don't bother adding */
2925                 if (!logical_len)
2926                         goto out;
2927         }
2928
2929         if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
2930                 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */
2931
2932                 /*
2933                  * For mwrite(mmap + memset to write) case, we still reserve
2934                  * space for NOCOW range.
2935                  * As NOCOW won't cause a new delayed ref, just free the space
2936                  */
2937                 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset,
2938                                        ordered_extent->len);
2939                 btrfs_ordered_update_i_size(inode, 0, ordered_extent);
2940                 if (nolock)
2941                         trans = btrfs_join_transaction_nolock(root);
2942                 else
2943                         trans = btrfs_join_transaction(root);
2944                 if (IS_ERR(trans)) {
2945                         ret = PTR_ERR(trans);
2946                         trans = NULL;
2947                         goto out;
2948                 }
2949                 trans->block_rsv = &fs_info->delalloc_block_rsv;
2950                 ret = btrfs_update_inode_fallback(trans, root, inode);
2951                 if (ret) /* -ENOMEM or corruption */
2952                         btrfs_abort_transaction(trans, ret);
2953                 goto out;
2954         }
2955
2956         range_locked = true;
2957         lock_extent_bits(io_tree, ordered_extent->file_offset,
2958                          ordered_extent->file_offset + ordered_extent->len - 1,
2959                          &cached_state);
2960
2961         ret = test_range_bit(io_tree, ordered_extent->file_offset,
2962                         ordered_extent->file_offset + ordered_extent->len - 1,
2963                         EXTENT_DEFRAG, 0, cached_state);
2964         if (ret) {
2965                 u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item);
2966                 if (0 && last_snapshot >= BTRFS_I(inode)->generation)
2967                         /* the inode is shared */
2968                         new = record_old_file_extents(inode, ordered_extent);
2969
2970                 clear_extent_bit(io_tree, ordered_extent->file_offset,
2971                         ordered_extent->file_offset + ordered_extent->len - 1,
2972                         EXTENT_DEFRAG, 0, 0, &cached_state, GFP_NOFS);
2973         }
2974
2975         if (nolock)
2976                 trans = btrfs_join_transaction_nolock(root);
2977         else
2978                 trans = btrfs_join_transaction(root);
2979         if (IS_ERR(trans)) {
2980                 ret = PTR_ERR(trans);
2981                 trans = NULL;
2982                 goto out;
2983         }
2984
2985         trans->block_rsv = &fs_info->delalloc_block_rsv;
2986
2987         if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
2988                 compress_type = ordered_extent->compress_type;
2989         if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
2990                 BUG_ON(compress_type);
2991                 ret = btrfs_mark_extent_written(trans, BTRFS_I(inode),
2992                                                 ordered_extent->file_offset,
2993                                                 ordered_extent->file_offset +
2994                                                 logical_len);
2995         } else {
2996                 BUG_ON(root == fs_info->tree_root);
2997                 ret = insert_reserved_file_extent(trans, inode,
2998                                                 ordered_extent->file_offset,
2999                                                 ordered_extent->start,
3000                                                 ordered_extent->disk_len,
3001                                                 logical_len, logical_len,
3002                                                 compress_type, 0, 0,
3003                                                 BTRFS_FILE_EXTENT_REG);
3004                 if (!ret)
3005                         btrfs_release_delalloc_bytes(fs_info,
3006                                                      ordered_extent->start,
3007                                                      ordered_extent->disk_len);
3008         }
3009         unpin_extent_cache(&BTRFS_I(inode)->extent_tree,
3010                            ordered_extent->file_offset, ordered_extent->len,
3011                            trans->transid);
3012         if (ret < 0) {
3013                 btrfs_abort_transaction(trans, ret);
3014                 goto out;
3015         }
3016
3017         add_pending_csums(trans, inode, &ordered_extent->list);
3018
3019         btrfs_ordered_update_i_size(inode, 0, ordered_extent);
3020         ret = btrfs_update_inode_fallback(trans, root, inode);
3021         if (ret) { /* -ENOMEM or corruption */
3022                 btrfs_abort_transaction(trans, ret);
3023                 goto out;
3024         }
3025         ret = 0;
3026 out:
3027         if (range_locked || clear_new_delalloc_bytes) {
3028                 unsigned int clear_bits = 0;
3029
3030                 if (range_locked)
3031                         clear_bits |= EXTENT_LOCKED;
3032                 if (clear_new_delalloc_bytes)
3033                         clear_bits |= EXTENT_DELALLOC_NEW;
3034                 clear_extent_bit(&BTRFS_I(inode)->io_tree,
3035                                  ordered_extent->file_offset,
3036                                  ordered_extent->file_offset +
3037                                  ordered_extent->len - 1,
3038                                  clear_bits,
3039                                  (clear_bits & EXTENT_LOCKED) ? 1 : 0,
3040                                  0, &cached_state, GFP_NOFS);
3041         }
3042
3043         if (root != fs_info->tree_root)
3044                 btrfs_delalloc_release_metadata(BTRFS_I(inode),
3045                                 ordered_extent->len);
3046         if (trans)
3047                 btrfs_end_transaction(trans);
3048
3049         if (ret || truncated) {
3050                 u64 start, end;
3051
3052                 if (truncated)
3053                         start = ordered_extent->file_offset + logical_len;
3054                 else
3055                         start = ordered_extent->file_offset;
3056                 end = ordered_extent->file_offset + ordered_extent->len - 1;
3057                 clear_extent_uptodate(io_tree, start, end, NULL, GFP_NOFS);
3058
3059                 /* Drop the cache for the part of the extent we didn't write. */
3060                 btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 0);
3061
3062                 /*
3063                  * If the ordered extent had an IOERR or something else went
3064                  * wrong we need to return the space for this ordered extent
3065                  * back to the allocator.  We only free the extent in the
3066                  * truncated case if we didn't write out the extent at all.
3067                  */
3068                 if ((ret || !logical_len) &&
3069                     !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3070                     !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags))
3071                         btrfs_free_reserved_extent(fs_info,
3072                                                    ordered_extent->start,
3073                                                    ordered_extent->disk_len, 1);
3074         }
3075
3076
3077         /*
3078          * This needs to be done to make sure anybody waiting knows we are done
3079          * updating everything for this ordered extent.
3080          */
3081         btrfs_remove_ordered_extent(inode, ordered_extent);
3082
3083         /* for snapshot-aware defrag */
3084         if (new) {
3085                 if (ret) {
3086                         free_sa_defrag_extent(new);
3087                         atomic_dec(&fs_info->defrag_running);
3088                 } else {
3089                         relink_file_extents(new);
3090                 }
3091         }
3092
3093         /* once for us */
3094         btrfs_put_ordered_extent(ordered_extent);
3095         /* once for the tree */
3096         btrfs_put_ordered_extent(ordered_extent);
3097
3098         return ret;
3099 }
3100
3101 static void finish_ordered_fn(struct btrfs_work *work)
3102 {
3103         struct btrfs_ordered_extent *ordered_extent;
3104         ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
3105         btrfs_finish_ordered_io(ordered_extent);
3106 }
3107
3108 static void btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end,
3109                                 struct extent_state *state, int uptodate)
3110 {
3111         struct inode *inode = page->mapping->host;
3112         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3113         struct btrfs_ordered_extent *ordered_extent = NULL;
3114         struct btrfs_workqueue *wq;
3115         btrfs_work_func_t func;
3116
3117         trace_btrfs_writepage_end_io_hook(page, start, end, uptodate);
3118
3119         ClearPagePrivate2(page);
3120         if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start,
3121                                             end - start + 1, uptodate))
3122                 return;
3123
3124         if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
3125                 wq = fs_info->endio_freespace_worker;
3126                 func = btrfs_freespace_write_helper;
3127         } else {
3128                 wq = fs_info->endio_write_workers;
3129                 func = btrfs_endio_write_helper;
3130         }
3131
3132         btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL,
3133                         NULL);
3134         btrfs_queue_work(wq, &ordered_extent->work);
3135 }
3136
3137 static int __readpage_endio_check(struct inode *inode,
3138                                   struct btrfs_io_bio *io_bio,
3139                                   int icsum, struct page *page,
3140                                   int pgoff, u64 start, size_t len)
3141 {
3142         char *kaddr;
3143         u32 csum_expected;
3144         u32 csum = ~(u32)0;
3145
3146         csum_expected = *(((u32 *)io_bio->csum) + icsum);
3147
3148         kaddr = kmap_atomic(page);
3149         csum = btrfs_csum_data(kaddr + pgoff, csum,  len);
3150         btrfs_csum_final(csum, (u8 *)&csum);
3151         if (csum != csum_expected)
3152                 goto zeroit;
3153
3154         kunmap_atomic(kaddr);
3155         return 0;
3156 zeroit:
3157         btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected,
3158                                     io_bio->mirror_num);
3159         memset(kaddr + pgoff, 1, len);
3160         flush_dcache_page(page);
3161         kunmap_atomic(kaddr);
3162         return -EIO;
3163 }
3164
3165 /*
3166  * when reads are done, we need to check csums to verify the data is correct
3167  * if there's a match, we allow the bio to finish.  If not, the code in
3168  * extent_io.c will try to find good copies for us.
3169  */
3170 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
3171                                       u64 phy_offset, struct page *page,
3172                                       u64 start, u64 end, int mirror)
3173 {
3174         size_t offset = start - page_offset(page);
3175         struct inode *inode = page->mapping->host;
3176         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
3177         struct btrfs_root *root = BTRFS_I(inode)->root;
3178
3179         if (PageChecked(page)) {
3180                 ClearPageChecked(page);
3181                 return 0;
3182         }
3183
3184         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
3185                 return 0;
3186
3187         if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID &&
3188             test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) {
3189                 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM);
3190                 return 0;
3191         }
3192
3193         phy_offset >>= inode->i_sb->s_blocksize_bits;
3194         return __readpage_endio_check(inode, io_bio, phy_offset, page, offset,
3195                                       start, (size_t)(end - start + 1));
3196 }
3197
3198 void btrfs_add_delayed_iput(struct inode *inode)
3199 {
3200         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3201         struct btrfs_inode *binode = BTRFS_I(inode);
3202
3203         if (atomic_add_unless(&inode->i_count, -1, 1))
3204                 return;
3205
3206         spin_lock(&fs_info->delayed_iput_lock);
3207         if (binode->delayed_iput_count == 0) {
3208                 ASSERT(list_empty(&binode->delayed_iput));
3209                 list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs);
3210         } else {
3211                 binode->delayed_iput_count++;
3212         }
3213         spin_unlock(&fs_info->delayed_iput_lock);
3214 }
3215
3216 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info)
3217 {
3218
3219         spin_lock(&fs_info->delayed_iput_lock);
3220         while (!list_empty(&fs_info->delayed_iputs)) {
3221                 struct btrfs_inode *inode;
3222
3223                 inode = list_first_entry(&fs_info->delayed_iputs,
3224                                 struct btrfs_inode, delayed_iput);
3225                 if (inode->delayed_iput_count) {
3226                         inode->delayed_iput_count--;
3227                         list_move_tail(&inode->delayed_iput,
3228                                         &fs_info->delayed_iputs);
3229                 } else {
3230                         list_del_init(&inode->delayed_iput);
3231                 }
3232                 spin_unlock(&fs_info->delayed_iput_lock);
3233                 iput(&inode->vfs_inode);
3234                 spin_lock(&fs_info->delayed_iput_lock);
3235         }
3236         spin_unlock(&fs_info->delayed_iput_lock);
3237 }
3238
3239 /*
3240  * This is called in transaction commit time. If there are no orphan
3241  * files in the subvolume, it removes orphan item and frees block_rsv
3242  * structure.
3243  */
3244 void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans,
3245                               struct btrfs_root *root)
3246 {
3247         struct btrfs_fs_info *fs_info = root->fs_info;
3248         struct btrfs_block_rsv *block_rsv;
3249         int ret;
3250
3251         if (atomic_read(&root->orphan_inodes) ||
3252             root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE)
3253                 return;
3254
3255         spin_lock(&root->orphan_lock);
3256         if (atomic_read(&root->orphan_inodes)) {
3257                 spin_unlock(&root->orphan_lock);
3258                 return;
3259         }
3260
3261         if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) {
3262                 spin_unlock(&root->orphan_lock);
3263                 return;
3264         }
3265
3266         block_rsv = root->orphan_block_rsv;
3267         root->orphan_block_rsv = NULL;
3268         spin_unlock(&root->orphan_lock);
3269
3270         if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) &&
3271             btrfs_root_refs(&root->root_item) > 0) {
3272                 ret = btrfs_del_orphan_item(trans, fs_info->tree_root,
3273                                             root->root_key.objectid);
3274                 if (ret)
3275                         btrfs_abort_transaction(trans, ret);
3276                 else
3277                         clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED,
3278                                   &root->state);
3279         }
3280
3281         if (block_rsv) {
3282                 WARN_ON(block_rsv->size > 0);
3283                 btrfs_free_block_rsv(fs_info, block_rsv);
3284         }
3285 }
3286
3287 /*
3288  * This creates an orphan entry for the given inode in case something goes
3289  * wrong in the middle of an unlink/truncate.
3290  *
3291  * NOTE: caller of this function should reserve 5 units of metadata for
3292  *       this function.
3293  */
3294 int btrfs_orphan_add(struct btrfs_trans_handle *trans,
3295                 struct btrfs_inode *inode)
3296 {
3297         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
3298         struct btrfs_root *root = inode->root;
3299         struct btrfs_block_rsv *block_rsv = NULL;
3300         int reserve = 0;
3301         int insert = 0;
3302         int ret;
3303
3304         if (!root->orphan_block_rsv) {
3305                 block_rsv = btrfs_alloc_block_rsv(fs_info,
3306                                                   BTRFS_BLOCK_RSV_TEMP);
3307                 if (!block_rsv)
3308                         return -ENOMEM;
3309         }
3310
3311         spin_lock(&root->orphan_lock);
3312         if (!root->orphan_block_rsv) {
3313                 root->orphan_block_rsv = block_rsv;
3314         } else if (block_rsv) {
3315                 btrfs_free_block_rsv(fs_info, block_rsv);
3316                 block_rsv = NULL;
3317         }
3318
3319         if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3320                               &inode->runtime_flags)) {
3321 #if 0
3322                 /*
3323                  * For proper ENOSPC handling, we should do orphan
3324                  * cleanup when mounting. But this introduces backward
3325                  * compatibility issue.
3326                  */
3327                 if (!xchg(&root->orphan_item_inserted, 1))
3328                         insert = 2;
3329                 else
3330                         insert = 1;
3331 #endif
3332                 insert = 1;
3333                 atomic_inc(&root->orphan_inodes);
3334         }
3335
3336         if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3337                               &inode->runtime_flags))
3338                 reserve = 1;
3339         spin_unlock(&root->orphan_lock);
3340
3341         /* grab metadata reservation from transaction handle */
3342         if (reserve) {
3343                 ret = btrfs_orphan_reserve_metadata(trans, inode);
3344                 ASSERT(!ret);
3345                 if (ret) {
3346                         atomic_dec(&root->orphan_inodes);
3347                         clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3348                                   &inode->runtime_flags);
3349                         if (insert)
3350                                 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3351                                           &inode->runtime_flags);
3352                         return ret;
3353                 }
3354         }
3355
3356         /* insert an orphan item to track this unlinked/truncated file */
3357         if (insert >= 1) {
3358                 ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode));
3359                 if (ret) {
3360                         atomic_dec(&root->orphan_inodes);
3361                         if (reserve) {
3362                                 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3363                                           &inode->runtime_flags);
3364                                 btrfs_orphan_release_metadata(inode);
3365                         }
3366                         if (ret != -EEXIST) {
3367                                 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3368                                           &inode->runtime_flags);
3369                                 btrfs_abort_transaction(trans, ret);
3370                                 return ret;
3371                         }
3372                 }
3373                 ret = 0;
3374         }
3375
3376         /* insert an orphan item to track subvolume contains orphan files */
3377         if (insert >= 2) {
3378                 ret = btrfs_insert_orphan_item(trans, fs_info->tree_root,
3379                                                root->root_key.objectid);
3380                 if (ret && ret != -EEXIST) {
3381                         btrfs_abort_transaction(trans, ret);
3382                         return ret;
3383                 }
3384         }
3385         return 0;
3386 }
3387
3388 /*
3389  * We have done the truncate/delete so we can go ahead and remove the orphan
3390  * item for this particular inode.
3391  */
3392 static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
3393                             struct btrfs_inode *inode)
3394 {
3395         struct btrfs_root *root = inode->root;
3396         int delete_item = 0;
3397         int release_rsv = 0;
3398         int ret = 0;
3399
3400         spin_lock(&root->orphan_lock);
3401         if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3402                                &inode->runtime_flags))
3403                 delete_item = 1;
3404
3405         if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3406                                &inode->runtime_flags))
3407                 release_rsv = 1;
3408         spin_unlock(&root->orphan_lock);
3409
3410         if (delete_item) {
3411                 atomic_dec(&root->orphan_inodes);
3412                 if (trans)
3413                         ret = btrfs_del_orphan_item(trans, root,
3414                                                     btrfs_ino(inode));
3415         }
3416
3417         if (release_rsv)
3418                 btrfs_orphan_release_metadata(inode);
3419
3420         return ret;
3421 }
3422
3423 /*
3424  * this cleans up any orphans that may be left on the list from the last use
3425  * of this root.
3426  */
3427 int btrfs_orphan_cleanup(struct btrfs_root *root)
3428 {
3429         struct btrfs_fs_info *fs_info = root->fs_info;
3430         struct btrfs_path *path;
3431         struct extent_buffer *leaf;
3432         struct btrfs_key key, found_key;
3433         struct btrfs_trans_handle *trans;
3434         struct inode *inode;
3435         u64 last_objectid = 0;
3436         int ret = 0, nr_unlink = 0, nr_truncate = 0;
3437
3438         if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED))
3439                 return 0;
3440
3441         path = btrfs_alloc_path();
3442         if (!path) {
3443                 ret = -ENOMEM;
3444                 goto out;
3445         }
3446         path->reada = READA_BACK;
3447
3448         key.objectid = BTRFS_ORPHAN_OBJECTID;
3449         key.type = BTRFS_ORPHAN_ITEM_KEY;
3450         key.offset = (u64)-1;
3451
3452         while (1) {
3453                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3454                 if (ret < 0)
3455                         goto out;
3456
3457                 /*
3458                  * if ret == 0 means we found what we were searching for, which
3459                  * is weird, but possible, so only screw with path if we didn't
3460                  * find the key and see if we have stuff that matches
3461                  */
3462                 if (ret > 0) {
3463                         ret = 0;
3464                         if (path->slots[0] == 0)
3465                                 break;
3466                         path->slots[0]--;
3467                 }
3468
3469                 /* pull out the item */
3470                 leaf = path->nodes[0];
3471                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3472
3473                 /* make sure the item matches what we want */
3474                 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
3475                         break;
3476                 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
3477                         break;
3478
3479                 /* release the path since we're done with it */
3480                 btrfs_release_path(path);
3481
3482                 /*
3483                  * this is where we are basically btrfs_lookup, without the
3484                  * crossing root thing.  we store the inode number in the
3485                  * offset of the orphan item.
3486                  */
3487
3488                 if (found_key.offset == last_objectid) {
3489                         btrfs_err(fs_info,
3490                                   "Error removing orphan entry, stopping orphan cleanup");
3491                         ret = -EINVAL;
3492                         goto out;
3493                 }
3494
3495                 last_objectid = found_key.offset;
3496
3497                 found_key.objectid = found_key.offset;
3498                 found_key.type = BTRFS_INODE_ITEM_KEY;
3499                 found_key.offset = 0;
3500                 inode = btrfs_iget(fs_info->sb, &found_key, root, NULL);
3501                 ret = PTR_ERR_OR_ZERO(inode);
3502                 if (ret && ret != -ENOENT)
3503                         goto out;
3504
3505                 if (ret == -ENOENT && root == fs_info->tree_root) {
3506                         struct btrfs_root *dead_root;
3507                         struct btrfs_fs_info *fs_info = root->fs_info;
3508                         int is_dead_root = 0;
3509
3510                         /*
3511                          * this is an orphan in the tree root. Currently these
3512                          * could come from 2 sources:
3513                          *  a) a snapshot deletion in progress
3514                          *  b) a free space cache inode
3515                          * We need to distinguish those two, as the snapshot
3516                          * orphan must not get deleted.
3517                          * find_dead_roots already ran before us, so if this
3518                          * is a snapshot deletion, we should find the root
3519                          * in the dead_roots list
3520                          */
3521                         spin_lock(&fs_info->trans_lock);
3522                         list_for_each_entry(dead_root, &fs_info->dead_roots,
3523                                             root_list) {
3524                                 if (dead_root->root_key.objectid ==
3525                                     found_key.objectid) {
3526                                         is_dead_root = 1;
3527                                         break;
3528                                 }
3529                         }
3530                         spin_unlock(&fs_info->trans_lock);
3531                         if (is_dead_root) {
3532                                 /* prevent this orphan from being found again */
3533                                 key.offset = found_key.objectid - 1;
3534                                 continue;
3535                         }
3536                 }
3537                 /*
3538                  * Inode is already gone but the orphan item is still there,
3539                  * kill the orphan item.
3540                  */
3541                 if (ret == -ENOENT) {
3542                         trans = btrfs_start_transaction(root, 1);
3543                         if (IS_ERR(trans)) {
3544                                 ret = PTR_ERR(trans);
3545                                 goto out;
3546                         }
3547                         btrfs_debug(fs_info, "auto deleting %Lu",
3548                                     found_key.objectid);
3549                         ret = btrfs_del_orphan_item(trans, root,
3550                                                     found_key.objectid);
3551                         btrfs_end_transaction(trans);
3552                         if (ret)
3553                                 goto out;
3554                         continue;
3555                 }
3556
3557                 /*
3558                  * add this inode to the orphan list so btrfs_orphan_del does
3559                  * the proper thing when we hit it
3560                  */
3561                 set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3562                         &BTRFS_I(inode)->runtime_flags);
3563                 atomic_inc(&root->orphan_inodes);
3564
3565                 /* if we have links, this was a truncate, lets do that */
3566                 if (inode->i_nlink) {
3567                         if (WARN_ON(!S_ISREG(inode->i_mode))) {
3568                                 iput(inode);
3569                                 continue;
3570                         }
3571                         nr_truncate++;
3572
3573                         /* 1 for the orphan item deletion. */
3574                         trans = btrfs_start_transaction(root, 1);
3575                         if (IS_ERR(trans)) {
3576                                 iput(inode);
3577                                 ret = PTR_ERR(trans);
3578                                 goto out;
3579                         }
3580                         ret = btrfs_orphan_add(trans, BTRFS_I(inode));
3581                         btrfs_end_transaction(trans);
3582                         if (ret) {
3583                                 iput(inode);
3584                                 goto out;
3585                         }
3586
3587                         ret = btrfs_truncate(inode);
3588                         if (ret)
3589                                 btrfs_orphan_del(NULL, BTRFS_I(inode));
3590                 } else {
3591                         nr_unlink++;
3592                 }
3593
3594                 /* this will do delete_inode and everything for us */
3595                 iput(inode);
3596                 if (ret)
3597                         goto out;
3598         }
3599         /* release the path since we're done with it */
3600         btrfs_release_path(path);
3601
3602         root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE;
3603
3604         if (root->orphan_block_rsv)
3605                 btrfs_block_rsv_release(fs_info, root->orphan_block_rsv,
3606                                         (u64)-1);
3607
3608         if (root->orphan_block_rsv ||
3609             test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
3610                 trans = btrfs_join_transaction(root);
3611                 if (!IS_ERR(trans))
3612                         btrfs_end_transaction(trans);
3613         }
3614
3615         if (nr_unlink)
3616                 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink);
3617         if (nr_truncate)
3618                 btrfs_debug(fs_info, "truncated %d orphans", nr_truncate);
3619
3620 out:
3621         if (ret)
3622                 btrfs_err(fs_info, "could not do orphan cleanup %d", ret);
3623         btrfs_free_path(path);
3624         return ret;
3625 }
3626
3627 /*
3628  * very simple check to peek ahead in the leaf looking for xattrs.  If we
3629  * don't find any xattrs, we know there can't be any acls.
3630  *
3631  * slot is the slot the inode is in, objectid is the objectid of the inode
3632  */
3633 static noinline int acls_after_inode_item(struct extent_buffer *leaf,
3634                                           int slot, u64 objectid,
3635                                           int *first_xattr_slot)
3636 {
3637         u32 nritems = btrfs_header_nritems(leaf);
3638         struct btrfs_key found_key;
3639         static u64 xattr_access = 0;
3640         static u64 xattr_default = 0;
3641         int scanned = 0;
3642
3643         if (!xattr_access) {
3644                 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,
3645                                         strlen(XATTR_NAME_POSIX_ACL_ACCESS));
3646                 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,
3647                                         strlen(XATTR_NAME_POSIX_ACL_DEFAULT));
3648         }
3649
3650         slot++;
3651         *first_xattr_slot = -1;
3652         while (slot < nritems) {
3653                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
3654
3655                 /* we found a different objectid, there must not be acls */
3656                 if (found_key.objectid != objectid)
3657                         return 0;
3658
3659                 /* we found an xattr, assume we've got an acl */
3660                 if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
3661                         if (*first_xattr_slot == -1)
3662                                 *first_xattr_slot = slot;
3663                         if (found_key.offset == xattr_access ||
3664                             found_key.offset == xattr_default)
3665                                 return 1;
3666                 }
3667
3668                 /*
3669                  * we found a key greater than an xattr key, there can't
3670                  * be any acls later on
3671                  */
3672                 if (found_key.type > BTRFS_XATTR_ITEM_KEY)
3673                         return 0;
3674
3675                 slot++;
3676                 scanned++;
3677
3678                 /*
3679                  * it goes inode, inode backrefs, xattrs, extents,
3680                  * so if there are a ton of hard links to an inode there can
3681                  * be a lot of backrefs.  Don't waste time searching too hard,
3682                  * this is just an optimization
3683                  */
3684                 if (scanned >= 8)
3685                         break;
3686         }
3687         /* we hit the end of the leaf before we found an xattr or
3688          * something larger than an xattr.  We have to assume the inode
3689          * has acls
3690          */
3691         if (*first_xattr_slot == -1)
3692                 *first_xattr_slot = slot;
3693         return 1;
3694 }
3695
3696 /*
3697  * read an inode from the btree into the in-memory inode
3698  */
3699 static int btrfs_read_locked_inode(struct inode *inode)
3700 {
3701         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3702         struct btrfs_path *path;
3703         struct extent_buffer *leaf;
3704         struct btrfs_inode_item *inode_item;
3705         struct btrfs_root *root = BTRFS_I(inode)->root;
3706         struct btrfs_key location;
3707         unsigned long ptr;
3708         int maybe_acls;
3709         u32 rdev;
3710         int ret;
3711         bool filled = false;
3712         int first_xattr_slot;
3713
3714         ret = btrfs_fill_inode(inode, &rdev);
3715         if (!ret)
3716                 filled = true;
3717
3718         path = btrfs_alloc_path();
3719         if (!path) {
3720                 ret = -ENOMEM;
3721                 goto make_bad;
3722         }
3723
3724         memcpy(&location, &BTRFS_I(inode)->location, sizeof(location));
3725
3726         ret = btrfs_lookup_inode(NULL, root, path, &location, 0);
3727         if (ret) {
3728                 if (ret > 0)
3729                         ret = -ENOENT;
3730                 goto make_bad;
3731         }
3732
3733         leaf = path->nodes[0];
3734
3735         if (filled)
3736                 goto cache_index;
3737
3738         inode_item = btrfs_item_ptr(leaf, path->slots[0],
3739                                     struct btrfs_inode_item);
3740         inode->i_mode = btrfs_inode_mode(leaf, inode_item);
3741         set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));
3742         i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));
3743         i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));
3744         btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item));
3745
3746         inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime);
3747         inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime);
3748
3749         inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime);
3750         inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime);
3751
3752         inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime);
3753         inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime);
3754
3755         BTRFS_I(inode)->i_otime.tv_sec =
3756                 btrfs_timespec_sec(leaf, &inode_item->otime);
3757         BTRFS_I(inode)->i_otime.tv_nsec =
3758                 btrfs_timespec_nsec(leaf, &inode_item->otime);
3759
3760         inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item));
3761         BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item);
3762         BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item);
3763
3764         inode->i_version = btrfs_inode_sequence(leaf, inode_item);
3765         inode->i_generation = BTRFS_I(inode)->generation;
3766         inode->i_rdev = 0;
3767         rdev = btrfs_inode_rdev(leaf, inode_item);
3768
3769         BTRFS_I(inode)->index_cnt = (u64)-1;
3770         BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item);
3771
3772 cache_index:
3773         /*
3774          * If we were modified in the current generation and evicted from memory
3775          * and then re-read we need to do a full sync since we don't have any
3776          * idea about which extents were modified before we were evicted from
3777          * cache.
3778          *
3779          * This is required for both inode re-read from disk and delayed inode
3780          * in delayed_nodes_tree.
3781          */
3782         if (BTRFS_I(inode)->last_trans == fs_info->generation)
3783                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
3784                         &BTRFS_I(inode)->runtime_flags);
3785
3786         /*
3787          * We don't persist the id of the transaction where an unlink operation
3788          * against the inode was last made. So here we assume the inode might
3789          * have been evicted, and therefore the exact value of last_unlink_trans
3790          * lost, and set it to last_trans to avoid metadata inconsistencies
3791          * between the inode and its parent if the inode is fsync'ed and the log
3792          * replayed. For example, in the scenario:
3793          *
3794          * touch mydir/foo
3795          * ln mydir/foo mydir/bar
3796          * sync
3797          * unlink mydir/bar
3798          * echo 2 > /proc/sys/vm/drop_caches   # evicts inode
3799          * xfs_io -c fsync mydir/foo
3800          * <power failure>
3801          * mount fs, triggers fsync log replay
3802          *
3803          * We must make sure that when we fsync our inode foo we also log its
3804          * parent inode, otherwise after log replay the parent still has the
3805          * dentry with the "bar" name but our inode foo has a link count of 1
3806          * and doesn't have an inode ref with the name "bar" anymore.
3807          *
3808          * Setting last_unlink_trans to last_trans is a pessimistic approach,
3809          * but it guarantees correctness at the expense of occasional full
3810          * transaction commits on fsync if our inode is a directory, or if our
3811          * inode is not a directory, logging its parent unnecessarily.
3812          */
3813         BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans;
3814
3815         path->slots[0]++;
3816         if (inode->i_nlink != 1 ||
3817             path->slots[0] >= btrfs_header_nritems(leaf))
3818                 goto cache_acl;
3819
3820         btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);
3821         if (location.objectid != btrfs_ino(BTRFS_I(inode)))
3822                 goto cache_acl;
3823
3824         ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
3825         if (location.type == BTRFS_INODE_REF_KEY) {
3826                 struct btrfs_inode_ref *ref;
3827
3828                 ref = (struct btrfs_inode_ref *)ptr;
3829                 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref);
3830         } else if (location.type == BTRFS_INODE_EXTREF_KEY) {
3831                 struct btrfs_inode_extref *extref;
3832
3833                 extref = (struct btrfs_inode_extref *)ptr;
3834                 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf,
3835                                                                      extref);
3836         }
3837 cache_acl:
3838         /*
3839          * try to precache a NULL acl entry for files that don't have
3840          * any xattrs or acls
3841          */
3842         maybe_acls = acls_after_inode_item(leaf, path->slots[0],
3843                         btrfs_ino(BTRFS_I(inode)), &first_xattr_slot);
3844         if (first_xattr_slot != -1) {
3845                 path->slots[0] = first_xattr_slot;
3846                 ret = btrfs_load_inode_props(inode, path);
3847                 if (ret)
3848                         btrfs_err(fs_info,
3849                                   "error loading props for ino %llu (root %llu): %d",
3850                                   btrfs_ino(BTRFS_I(inode)),
3851                                   root->root_key.objectid, ret);
3852         }
3853         btrfs_free_path(path);
3854
3855         if (!maybe_acls)
3856                 cache_no_acl(inode);
3857
3858         switch (inode->i_mode & S_IFMT) {
3859         case S_IFREG:
3860                 inode->i_mapping->a_ops = &btrfs_aops;
3861                 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
3862                 inode->i_fop = &btrfs_file_operations;
3863                 inode->i_op = &btrfs_file_inode_operations;
3864                 break;
3865         case S_IFDIR:
3866                 inode->i_fop = &btrfs_dir_file_operations;
3867                 inode->i_op = &btrfs_dir_inode_operations;
3868                 break;
3869         case S_IFLNK:
3870                 inode->i_op = &btrfs_symlink_inode_operations;
3871                 inode_nohighmem(inode);
3872                 inode->i_mapping->a_ops = &btrfs_symlink_aops;
3873                 break;
3874         default:
3875                 inode->i_op = &btrfs_special_inode_operations;
3876                 init_special_inode(inode, inode->i_mode, rdev);
3877                 break;
3878         }
3879
3880         btrfs_update_iflags(inode);
3881         return 0;
3882
3883 make_bad:
3884         btrfs_free_path(path);
3885         make_bad_inode(inode);
3886         return ret;
3887 }
3888
3889 /*
3890  * given a leaf and an inode, copy the inode fields into the leaf
3891  */
3892 static void fill_inode_item(struct btrfs_trans_handle *trans,
3893                             struct extent_buffer *leaf,
3894                             struct btrfs_inode_item *item,
3895                             struct inode *inode)
3896 {
3897         struct btrfs_map_token token;
3898
3899         btrfs_init_map_token(&token);
3900
3901         btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token);
3902         btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token);
3903         btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size,
3904                                    &token);
3905         btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token);
3906         btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token);
3907
3908         btrfs_set_token_timespec_sec(leaf, &item->atime,
3909                                      inode->i_atime.tv_sec, &token);
3910         btrfs_set_token_timespec_nsec(leaf, &item->atime,
3911                                       inode->i_atime.tv_nsec, &token);
3912
3913         btrfs_set_token_timespec_sec(leaf, &item->mtime,
3914                                      inode->i_mtime.tv_sec, &token);
3915         btrfs_set_token_timespec_nsec(leaf, &item->mtime,
3916                                       inode->i_mtime.tv_nsec, &token);
3917
3918         btrfs_set_token_timespec_sec(leaf, &item->ctime,
3919                                      inode->i_ctime.tv_sec, &token);
3920         btrfs_set_token_timespec_nsec(leaf, &item->ctime,
3921                                       inode->i_ctime.tv_nsec, &token);
3922
3923         btrfs_set_token_timespec_sec(leaf, &item->otime,
3924                                      BTRFS_I(inode)->i_otime.tv_sec, &token);
3925         btrfs_set_token_timespec_nsec(leaf, &item->otime,
3926                                       BTRFS_I(inode)->i_otime.tv_nsec, &token);
3927
3928         btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode),
3929                                      &token);
3930         btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation,
3931                                          &token);
3932         btrfs_set_token_inode_sequence(leaf, item, inode->i_version, &token);
3933         btrfs_set_token_inode_transid(leaf, item, trans->transid, &token);
3934         btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token);
3935         btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token);
3936         btrfs_set_token_inode_block_group(leaf, item, 0, &token);
3937 }
3938
3939 /*
3940  * copy everything in the in-memory inode into the btree.
3941  */
3942 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
3943                                 struct btrfs_root *root, struct inode *inode)
3944 {
3945         struct btrfs_inode_item *inode_item;
3946         struct btrfs_path *path;
3947         struct extent_buffer *leaf;
3948         int ret;
3949
3950         path = btrfs_alloc_path();
3951         if (!path)
3952                 return -ENOMEM;
3953
3954         path->leave_spinning = 1;
3955         ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location,
3956                                  1);
3957         if (ret) {
3958                 if (ret > 0)
3959                         ret = -ENOENT;
3960                 goto failed;
3961         }
3962
3963         leaf = path->nodes[0];
3964         inode_item = btrfs_item_ptr(leaf, path->slots[0],
3965                                     struct btrfs_inode_item);
3966
3967         fill_inode_item(trans, leaf, inode_item, inode);
3968         btrfs_mark_buffer_dirty(leaf);
3969         btrfs_set_inode_last_trans(trans, inode);
3970         ret = 0;
3971 failed:
3972         btrfs_free_path(path);
3973         return ret;
3974 }
3975
3976 /*
3977  * copy everything in the in-memory inode into the btree.
3978  */
3979 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans,
3980                                 struct btrfs_root *root, struct inode *inode)
3981 {
3982         struct btrfs_fs_info *fs_info = root->fs_info;
3983         int ret;
3984
3985         /*
3986          * If the inode is a free space inode, we can deadlock during commit
3987          * if we put it into the delayed code.
3988          *
3989          * The data relocation inode should also be directly updated
3990          * without delay
3991          */
3992         if (!btrfs_is_free_space_inode(BTRFS_I(inode))
3993             && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
3994             && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
3995                 btrfs_update_root_times(trans, root);
3996
3997                 ret = btrfs_delayed_update_inode(trans, root, inode);
3998                 if (!ret)
3999                         btrfs_set_inode_last_trans(trans, inode);
4000                 return ret;
4001         }
4002
4003         return btrfs_update_inode_item(trans, root, inode);
4004 }
4005
4006 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
4007                                          struct btrfs_root *root,
4008                                          struct inode *inode)
4009 {
4010         int ret;
4011
4012         ret = btrfs_update_inode(trans, root, inode);
4013         if (ret == -ENOSPC)
4014                 return btrfs_update_inode_item(trans, root, inode);
4015         return ret;
4016 }
4017
4018 /*
4019  * unlink helper that gets used here in inode.c and in the tree logging
4020  * recovery code.  It remove a link in a directory with a given name, and
4021  * also drops the back refs in the inode to the directory
4022  */
4023 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4024                                 struct btrfs_root *root,
4025                                 struct btrfs_inode *dir,
4026                                 struct btrfs_inode *inode,
4027                                 const char *name, int name_len)
4028 {
4029         struct btrfs_fs_info *fs_info = root->fs_info;
4030         struct btrfs_path *path;
4031         int ret = 0;
4032         struct extent_buffer *leaf;
4033         struct btrfs_dir_item *di;
4034         struct btrfs_key key;
4035         u64 index;
4036         u64 ino = btrfs_ino(inode);
4037         u64 dir_ino = btrfs_ino(dir);
4038
4039         path = btrfs_alloc_path();
4040         if (!path) {
4041                 ret = -ENOMEM;
4042                 goto out;
4043         }
4044
4045         path->leave_spinning = 1;
4046         di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4047                                     name, name_len, -1);
4048         if (IS_ERR(di)) {
4049                 ret = PTR_ERR(di);
4050                 goto err;
4051         }
4052         if (!di) {
4053                 ret = -ENOENT;
4054                 goto err;
4055         }
4056         leaf = path->nodes[0];
4057         btrfs_dir_item_key_to_cpu(leaf, di, &key);
4058         ret = btrfs_delete_one_dir_name(trans, root, path, di);
4059         if (ret)
4060                 goto err;
4061         btrfs_release_path(path);
4062
4063         /*
4064          * If we don't have dir index, we have to get it by looking up
4065          * the inode ref, since we get the inode ref, remove it directly,
4066          * it is unnecessary to do delayed deletion.
4067          *
4068          * But if we have dir index, needn't search inode ref to get it.
4069          * Since the inode ref is close to the inode item, it is better
4070          * that we delay to delete it, and just do this deletion when
4071          * we update the inode item.
4072          */
4073         if (inode->dir_index) {
4074                 ret = btrfs_delayed_delete_inode_ref(inode);
4075                 if (!ret) {
4076                         index = inode->dir_index;
4077                         goto skip_backref;
4078                 }
4079         }
4080
4081         ret = btrfs_del_inode_ref(trans, root, name, name_len, ino,
4082                                   dir_ino, &index);
4083         if (ret) {
4084                 btrfs_info(fs_info,
4085                         "failed to delete reference to %.*s, inode %llu parent %llu",
4086                         name_len, name, ino, dir_ino);
4087                 btrfs_abort_transaction(trans, ret);
4088                 goto err;
4089         }
4090 skip_backref:
4091         ret = btrfs_delete_delayed_dir_index(trans, fs_info, dir, index);
4092         if (ret) {
4093                 btrfs_abort_transaction(trans, ret);
4094                 goto err;
4095         }
4096
4097         ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, inode,
4098                         dir_ino);
4099         if (ret != 0 && ret != -ENOENT) {
4100                 btrfs_abort_transaction(trans, ret);
4101                 goto err;
4102         }
4103
4104         ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, dir,
4105                         index);
4106         if (ret == -ENOENT)
4107                 ret = 0;
4108         else if (ret)
4109                 btrfs_abort_transaction(trans, ret);
4110 err:
4111         btrfs_free_path(path);
4112         if (ret)
4113                 goto out;
4114
4115         btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2);
4116         inode_inc_iversion(&inode->vfs_inode);
4117         inode_inc_iversion(&dir->vfs_inode);
4118         inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime =
4119                 dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode);
4120         ret = btrfs_update_inode(trans, root, &dir->vfs_inode);
4121 out:
4122         return ret;
4123 }
4124
4125 int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4126                        struct btrfs_root *root,
4127                        struct btrfs_inode *dir, struct btrfs_inode *inode,
4128                        const char *name, int name_len)
4129 {
4130         int ret;
4131         ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len);
4132         if (!ret) {
4133                 drop_nlink(&inode->vfs_inode);
4134                 ret = btrfs_update_inode(trans, root, &inode->vfs_inode);
4135         }
4136         return ret;
4137 }
4138
4139 /*
4140  * helper to start transaction for unlink and rmdir.
4141  *
4142  * unlink and rmdir are special in btrfs, they do not always free space, so
4143  * if we cannot make our reservations the normal way try and see if there is
4144  * plenty of slack room in the global reserve to migrate, otherwise we cannot
4145  * allow the unlink to occur.
4146  */
4147 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir)
4148 {
4149         struct btrfs_root *root = BTRFS_I(dir)->root;
4150
4151         /*
4152          * 1 for the possible orphan item
4153          * 1 for the dir item
4154          * 1 for the dir index
4155          * 1 for the inode ref
4156          * 1 for the inode
4157          */
4158         return btrfs_start_transaction_fallback_global_rsv(root, 5, 5);
4159 }
4160
4161 static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
4162 {
4163         struct btrfs_root *root = BTRFS_I(dir)->root;
4164         struct btrfs_trans_handle *trans;
4165         struct inode *inode = d_inode(dentry);
4166         int ret;
4167
4168         trans = __unlink_start_trans(dir);
4169         if (IS_ERR(trans))
4170                 return PTR_ERR(trans);
4171
4172         btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
4173                         0);
4174
4175         ret = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
4176                         BTRFS_I(d_inode(dentry)), dentry->d_name.name,
4177                         dentry->d_name.len);
4178         if (ret)
4179                 goto out;
4180
4181         if (inode->i_nlink == 0) {
4182                 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
4183                 if (ret)
4184                         goto out;
4185         }
4186
4187 out:
4188         btrfs_end_transaction(trans);
4189         btrfs_btree_balance_dirty(root->fs_info);
4190         return ret;
4191 }
4192
4193 int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
4194                         struct btrfs_root *root,
4195                         struct inode *dir, u64 objectid,
4196                         const char *name, int name_len)
4197 {
4198         struct btrfs_fs_info *fs_info = root->fs_info;
4199         struct btrfs_path *path;
4200         struct extent_buffer *leaf;
4201         struct btrfs_dir_item *di;
4202         struct btrfs_key key;
4203         u64 index;
4204         int ret;
4205         u64 dir_ino = btrfs_ino(BTRFS_I(dir));
4206
4207         path = btrfs_alloc_path();
4208         if (!path)
4209                 return -ENOMEM;
4210
4211         di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4212                                    name, name_len, -1);
4213         if (IS_ERR_OR_NULL(di)) {
4214                 if (!di)
4215                         ret = -ENOENT;
4216                 else
4217                         ret = PTR_ERR(di);
4218                 goto out;
4219         }
4220
4221         leaf = path->nodes[0];
4222         btrfs_dir_item_key_to_cpu(leaf, di, &key);
4223         WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);
4224         ret = btrfs_delete_one_dir_name(trans, root, path, di);
4225         if (ret) {
4226                 btrfs_abort_transaction(trans, ret);
4227                 goto out;
4228         }
4229         btrfs_release_path(path);
4230
4231         ret = btrfs_del_root_ref(trans, fs_info, objectid,
4232                                  root->root_key.objectid, dir_ino,
4233                                  &index, name, name_len);
4234         if (ret < 0) {
4235                 if (ret != -ENOENT) {
4236                         btrfs_abort_transaction(trans, ret);
4237                         goto out;
4238                 }
4239                 di = btrfs_search_dir_index_item(root, path, dir_ino,
4240                                                  name, name_len);
4241                 if (IS_ERR_OR_NULL(di)) {
4242                         if (!di)
4243                                 ret = -ENOENT;
4244                         else
4245                                 ret = PTR_ERR(di);
4246                         btrfs_abort_transaction(trans, ret);
4247                         goto out;
4248                 }
4249
4250                 leaf = path->nodes[0];
4251                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4252                 btrfs_release_path(path);
4253                 index = key.offset;
4254         }
4255         btrfs_release_path(path);
4256
4257         ret = btrfs_delete_delayed_dir_index(trans, fs_info, BTRFS_I(dir), index);
4258         if (ret) {
4259                 btrfs_abort_transaction(trans, ret);
4260                 goto out;
4261         }
4262
4263         btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2);
4264         inode_inc_iversion(dir);
4265         dir->i_mtime = dir->i_ctime = current_time(dir);
4266         ret = btrfs_update_inode_fallback(trans, root, dir);
4267         if (ret)
4268                 btrfs_abort_transaction(trans, ret);
4269 out:
4270         btrfs_free_path(path);
4271         return ret;
4272 }
4273
4274 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)
4275 {
4276         struct inode *inode = d_inode(dentry);
4277         int err = 0;
4278         struct btrfs_root *root = BTRFS_I(dir)->root;
4279         struct btrfs_trans_handle *trans;
4280         u64 last_unlink_trans;
4281
4282         if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)
4283                 return -ENOTEMPTY;
4284         if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID)
4285                 return -EPERM;
4286
4287         trans = __unlink_start_trans(dir);
4288         if (IS_ERR(trans))
4289                 return PTR_ERR(trans);
4290
4291         if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
4292                 err = btrfs_unlink_subvol(trans, root, dir,
4293                                           BTRFS_I(inode)->location.objectid,
4294                                           dentry->d_name.name,
4295                                           dentry->d_name.len);
4296                 goto out;
4297         }
4298
4299         err = btrfs_orphan_add(trans, BTRFS_I(inode));
4300         if (err)
4301                 goto out;
4302
4303         last_unlink_trans = BTRFS_I(inode)->last_unlink_trans;
4304
4305         /* now the directory is empty */
4306         err = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
4307                         BTRFS_I(d_inode(dentry)), dentry->d_name.name,
4308                         dentry->d_name.len);
4309         if (!err) {
4310                 btrfs_i_size_write(BTRFS_I(inode), 0);
4311                 /*
4312                  * Propagate the last_unlink_trans value of the deleted dir to
4313                  * its parent directory. This is to prevent an unrecoverable
4314                  * log tree in the case we do something like this:
4315                  * 1) create dir foo
4316                  * 2) create snapshot under dir foo
4317                  * 3) delete the snapshot
4318                  * 4) rmdir foo
4319                  * 5) mkdir foo
4320                  * 6) fsync foo or some file inside foo
4321                  */
4322                 if (last_unlink_trans >= trans->transid)
4323                         BTRFS_I(dir)->last_unlink_trans = last_unlink_trans;
4324         }
4325 out:
4326         btrfs_end_transaction(trans);
4327         btrfs_btree_balance_dirty(root->fs_info);
4328
4329         return err;
4330 }
4331
4332 static int truncate_space_check(struct btrfs_trans_handle *trans,
4333                                 struct btrfs_root *root,
4334                                 u64 bytes_deleted)
4335 {
4336         struct btrfs_fs_info *fs_info = root->fs_info;
4337         int ret;
4338
4339         /*
4340          * This is only used to apply pressure to the enospc system, we don't
4341          * intend to use this reservation at all.
4342          */
4343         bytes_deleted = btrfs_csum_bytes_to_leaves(fs_info, bytes_deleted);
4344         bytes_deleted *= fs_info->nodesize;
4345         ret = btrfs_block_rsv_add(root, &fs_info->trans_block_rsv,
4346                                   bytes_deleted, BTRFS_RESERVE_NO_FLUSH);
4347         if (!ret) {
4348                 trace_btrfs_space_reservation(fs_info, "transaction",
4349                                               trans->transid,
4350                                               bytes_deleted, 1);
4351                 trans->bytes_reserved += bytes_deleted;
4352         }
4353         return ret;
4354
4355 }
4356
4357 static int truncate_inline_extent(struct inode *inode,
4358                                   struct btrfs_path *path,
4359                                   struct btrfs_key *found_key,
4360                                   const u64 item_end,
4361                                   const u64 new_size)
4362 {
4363         struct extent_buffer *leaf = path->nodes[0];
4364         int slot = path->slots[0];
4365         struct btrfs_file_extent_item *fi;
4366         u32 size = (u32)(new_size - found_key->offset);
4367         struct btrfs_root *root = BTRFS_I(inode)->root;
4368
4369         fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
4370
4371         if (btrfs_file_extent_compression(leaf, fi) != BTRFS_COMPRESS_NONE) {
4372                 loff_t offset = new_size;
4373                 loff_t page_end = ALIGN(offset, PAGE_SIZE);
4374
4375                 /*
4376                  * Zero out the remaining of the last page of our inline extent,
4377                  * instead of directly truncating our inline extent here - that
4378                  * would be much more complex (decompressing all the data, then
4379                  * compressing the truncated data, which might be bigger than
4380                  * the size of the inline extent, resize the extent, etc).
4381                  * We release the path because to get the page we might need to
4382                  * read the extent item from disk (data not in the page cache).
4383                  */
4384                 btrfs_release_path(path);
4385                 return btrfs_truncate_block(inode, offset, page_end - offset,
4386                                         0);
4387         }
4388
4389         btrfs_set_file_extent_ram_bytes(leaf, fi, size);
4390         size = btrfs_file_extent_calc_inline_size(size);
4391         btrfs_truncate_item(root->fs_info, path, size, 1);
4392
4393         if (test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4394                 inode_sub_bytes(inode, item_end + 1 - new_size);
4395
4396         return 0;
4397 }
4398
4399 /*
4400  * this can truncate away extent items, csum items and directory items.
4401  * It starts at a high offset and removes keys until it can't find
4402  * any higher than new_size
4403  *
4404  * csum items that cross the new i_size are truncated to the new size
4405  * as well.
4406  *
4407  * min_type is the minimum key type to truncate down to.  If set to 0, this
4408  * will kill all the items on this inode, including the INODE_ITEM_KEY.
4409  */
4410 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
4411                                struct btrfs_root *root,
4412                                struct inode *inode,
4413                                u64 new_size, u32 min_type)
4414 {
4415         struct btrfs_fs_info *fs_info = root->fs_info;
4416         struct btrfs_path *path;
4417         struct extent_buffer *leaf;
4418         struct btrfs_file_extent_item *fi;
4419         struct btrfs_key key;
4420         struct btrfs_key found_key;
4421         u64 extent_start = 0;
4422         u64 extent_num_bytes = 0;
4423         u64 extent_offset = 0;
4424         u64 item_end = 0;
4425         u64 last_size = new_size;
4426         u32 found_type = (u8)-1;
4427         int found_extent;
4428         int del_item;
4429         int pending_del_nr = 0;
4430         int pending_del_slot = 0;
4431         int extent_type = -1;
4432         int ret;
4433         int err = 0;
4434         u64 ino = btrfs_ino(BTRFS_I(inode));
4435         u64 bytes_deleted = 0;
4436         bool be_nice = 0;
4437         bool should_throttle = 0;
4438         bool should_end = 0;
4439
4440         BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY);
4441
4442         /*
4443          * for non-free space inodes and ref cows, we want to back off from
4444          * time to time
4445          */
4446         if (!btrfs_is_free_space_inode(BTRFS_I(inode)) &&
4447             test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4448                 be_nice = 1;
4449
4450         path = btrfs_alloc_path();
4451         if (!path)
4452                 return -ENOMEM;
4453         path->reada = READA_BACK;
4454
4455         /*
4456          * We want to drop from the next block forward in case this new size is
4457          * not block aligned since we will be keeping the last block of the
4458          * extent just the way it is.
4459          */
4460         if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4461             root == fs_info->tree_root)
4462                 btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size,
4463                                         fs_info->sectorsize),
4464                                         (u64)-1, 0);
4465
4466         /*
4467          * This function is also used to drop the items in the log tree before
4468          * we relog the inode, so if root != BTRFS_I(inode)->root, it means
4469          * it is used to drop the loged items. So we shouldn't kill the delayed
4470          * items.
4471          */
4472         if (min_type == 0 && root == BTRFS_I(inode)->root)
4473                 btrfs_kill_delayed_inode_items(BTRFS_I(inode));
4474
4475         key.objectid = ino;
4476         key.offset = (u64)-1;
4477         key.type = (u8)-1;
4478
4479 search_again:
4480         /*
4481          * with a 16K leaf size and 128MB extents, you can actually queue
4482          * up a huge file in a single leaf.  Most of the time that
4483          * bytes_deleted is > 0, it will be huge by the time we get here
4484          */
4485         if (be_nice && bytes_deleted > SZ_32M) {
4486                 if (btrfs_should_end_transaction(trans)) {
4487                         err = -EAGAIN;
4488                         goto error;
4489                 }
4490         }
4491
4492
4493         path->leave_spinning = 1;
4494         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
4495         if (ret < 0) {
4496                 err = ret;
4497                 goto out;
4498         }
4499
4500         if (ret > 0) {
4501                 /* there are no items in the tree for us to truncate, we're
4502                  * done
4503                  */
4504                 if (path->slots[0] == 0)
4505                         goto out;
4506                 path->slots[0]--;
4507         }
4508
4509         while (1) {
4510                 fi = NULL;
4511                 leaf = path->nodes[0];
4512                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
4513                 found_type = found_key.type;
4514
4515                 if (found_key.objectid != ino)
4516                         break;
4517
4518                 if (found_type < min_type)
4519                         break;
4520
4521                 item_end = found_key.offset;
4522                 if (found_type == BTRFS_EXTENT_DATA_KEY) {
4523                         fi = btrfs_item_ptr(leaf, path->slots[0],
4524                                             struct btrfs_file_extent_item);
4525                         extent_type = btrfs_file_extent_type(leaf, fi);
4526                         if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4527                                 item_end +=
4528                                     btrfs_file_extent_num_bytes(leaf, fi);
4529
4530                                 trace_btrfs_truncate_show_fi_regular(
4531                                         BTRFS_I(inode), leaf, fi,
4532                                         found_key.offset);
4533                         } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4534                                 item_end += btrfs_file_extent_inline_len(leaf,
4535                                                          path->slots[0], fi);
4536
4537                                 trace_btrfs_truncate_show_fi_inline(
4538                                         BTRFS_I(inode), leaf, fi, path->slots[0],
4539                                         found_key.offset);
4540                         }
4541                         item_end--;
4542                 }
4543                 if (found_type > min_type) {
4544                         del_item = 1;
4545                 } else {
4546                         if (item_end < new_size)
4547                                 break;
4548                         if (found_key.offset >= new_size)
4549                                 del_item = 1;
4550                         else
4551                                 del_item = 0;
4552                 }
4553                 found_extent = 0;
4554                 /* FIXME, shrink the extent if the ref count is only 1 */
4555                 if (found_type != BTRFS_EXTENT_DATA_KEY)
4556                         goto delete;
4557
4558                 if (del_item)
4559                         last_size = found_key.offset;
4560                 else
4561                         last_size = new_size;
4562
4563                 if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4564                         u64 num_dec;
4565                         extent_start = btrfs_file_extent_disk_bytenr(leaf, fi);
4566                         if (!del_item) {
4567                                 u64 orig_num_bytes =
4568                                         btrfs_file_extent_num_bytes(leaf, fi);
4569                                 extent_num_bytes = ALIGN(new_size -
4570                                                 found_key.offset,
4571                                                 fs_info->sectorsize);
4572                                 btrfs_set_file_extent_num_bytes(leaf, fi,
4573                                                          extent_num_bytes);
4574                                 num_dec = (orig_num_bytes -
4575                                            extent_num_bytes);
4576                                 if (test_bit(BTRFS_ROOT_REF_COWS,
4577                                              &root->state) &&
4578                                     extent_start != 0)
4579                                         inode_sub_bytes(inode, num_dec);
4580                                 btrfs_mark_buffer_dirty(leaf);
4581                         } else {
4582                                 extent_num_bytes =
4583                                         btrfs_file_extent_disk_num_bytes(leaf,
4584                                                                          fi);
4585                                 extent_offset = found_key.offset -
4586                                         btrfs_file_extent_offset(leaf, fi);
4587
4588                                 /* FIXME blocksize != 4096 */
4589                                 num_dec = btrfs_file_extent_num_bytes(leaf, fi);
4590                                 if (extent_start != 0) {
4591                                         found_extent = 1;
4592                                         if (test_bit(BTRFS_ROOT_REF_COWS,
4593                                                      &root->state))
4594                                                 inode_sub_bytes(inode, num_dec);
4595                                 }
4596                         }
4597                 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4598                         /*
4599                          * we can't truncate inline items that have had
4600                          * special encodings
4601                          */
4602                         if (!del_item &&
4603                             btrfs_file_extent_encryption(leaf, fi) == 0 &&
4604                             btrfs_file_extent_other_encoding(leaf, fi) == 0) {
4605
4606                                 /*
4607                                  * Need to release path in order to truncate a
4608                                  * compressed extent. So delete any accumulated
4609                                  * extent items so far.
4610                                  */
4611                                 if (btrfs_file_extent_compression(leaf, fi) !=
4612                                     BTRFS_COMPRESS_NONE && pending_del_nr) {
4613                                         err = btrfs_del_items(trans, root, path,
4614                                                               pending_del_slot,
4615                                                               pending_del_nr);
4616                                         if (err) {
4617                                                 btrfs_abort_transaction(trans,
4618                                                                         err);
4619                                                 goto error;
4620                                         }
4621                                         pending_del_nr = 0;
4622                                 }
4623
4624                                 err = truncate_inline_extent(inode, path,
4625                                                              &found_key,
4626                                                              item_end,
4627                                                              new_size);
4628                                 if (err) {
4629                                         btrfs_abort_transaction(trans, err);
4630                                         goto error;
4631                                 }
4632                         } else if (test_bit(BTRFS_ROOT_REF_COWS,
4633                                             &root->state)) {
4634                                 inode_sub_bytes(inode, item_end + 1 - new_size);
4635                         }
4636                 }
4637 delete:
4638                 if (del_item) {
4639                         if (!pending_del_nr) {
4640                                 /* no pending yet, add ourselves */
4641                                 pending_del_slot = path->slots[0];
4642                                 pending_del_nr = 1;
4643                         } else if (pending_del_nr &&
4644                                    path->slots[0] + 1 == pending_del_slot) {
4645                                 /* hop on the pending chunk */
4646                                 pending_del_nr++;
4647                                 pending_del_slot = path->slots[0];
4648                         } else {
4649                                 BUG();
4650                         }
4651                 } else {
4652                         break;
4653                 }
4654                 should_throttle = 0;
4655
4656                 if (found_extent &&
4657                     (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4658                      root == fs_info->tree_root)) {
4659                         btrfs_set_path_blocking(path);
4660                         bytes_deleted += extent_num_bytes;
4661                         ret = btrfs_free_extent(trans, fs_info, extent_start,
4662                                                 extent_num_bytes, 0,
4663                                                 btrfs_header_owner(leaf),
4664                                                 ino, extent_offset);
4665                         BUG_ON(ret);
4666                         if (btrfs_should_throttle_delayed_refs(trans, fs_info))
4667                                 btrfs_async_run_delayed_refs(fs_info,
4668                                         trans->delayed_ref_updates * 2,
4669                                         trans->transid, 0);
4670                         if (be_nice) {
4671                                 if (truncate_space_check(trans, root,
4672                                                          extent_num_bytes)) {
4673                                         should_end = 1;
4674                                 }
4675                                 if (btrfs_should_throttle_delayed_refs(trans,
4676                                                                        fs_info))
4677                                         should_throttle = 1;
4678                         }
4679                 }
4680
4681                 if (found_type == BTRFS_INODE_ITEM_KEY)
4682                         break;
4683
4684                 if (path->slots[0] == 0 ||
4685                     path->slots[0] != pending_del_slot ||
4686                     should_throttle || should_end) {
4687                         if (pending_del_nr) {
4688                                 ret = btrfs_del_items(trans, root, path,
4689                                                 pending_del_slot,
4690                                                 pending_del_nr);
4691                                 if (ret) {
4692                                         btrfs_abort_transaction(trans, ret);
4693                                         goto error;
4694                                 }
4695                                 pending_del_nr = 0;
4696                         }
4697                         btrfs_release_path(path);
4698                         if (should_throttle) {
4699                                 unsigned long updates = trans->delayed_ref_updates;
4700                                 if (updates) {
4701                                         trans->delayed_ref_updates = 0;
4702                                         ret = btrfs_run_delayed_refs(trans,
4703                                                                    fs_info,
4704                                                                    updates * 2);
4705                                         if (ret && !err)
4706                                                 err = ret;
4707                                 }
4708                         }
4709                         /*
4710                          * if we failed to refill our space rsv, bail out
4711                          * and let the transaction restart
4712                          */
4713                         if (should_end) {
4714                                 err = -EAGAIN;
4715                                 goto error;
4716                         }
4717                         goto search_again;
4718                 } else {
4719                         path->slots[0]--;
4720                 }
4721         }
4722 out:
4723         if (pending_del_nr) {
4724                 ret = btrfs_del_items(trans, root, path, pending_del_slot,
4725                                       pending_del_nr);
4726                 if (ret)
4727                         btrfs_abort_transaction(trans, ret);
4728         }
4729 error:
4730         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
4731                 ASSERT(last_size >= new_size);
4732                 if (!err && last_size > new_size)
4733                         last_size = new_size;
4734                 btrfs_ordered_update_i_size(inode, last_size, NULL);
4735         }
4736
4737         btrfs_free_path(path);
4738
4739         if (be_nice && bytes_deleted > SZ_32M) {
4740                 unsigned long updates = trans->delayed_ref_updates;
4741                 if (updates) {
4742                         trans->delayed_ref_updates = 0;
4743                         ret = btrfs_run_delayed_refs(trans, fs_info,
4744                                                      updates * 2);
4745                         if (ret && !err)
4746                                 err = ret;
4747                 }
4748         }
4749         return err;
4750 }
4751
4752 /*
4753  * btrfs_truncate_block - read, zero a chunk and write a block
4754  * @inode - inode that we're zeroing
4755  * @from - the offset to start zeroing
4756  * @len - the length to zero, 0 to zero the entire range respective to the
4757  *      offset
4758  * @front - zero up to the offset instead of from the offset on
4759  *
4760  * This will find the block for the "from" offset and cow the block and zero the
4761  * part we want to zero.  This is used with truncate and hole punching.
4762  */
4763 int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len,
4764                         int front)
4765 {
4766         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4767         struct address_space *mapping = inode->i_mapping;
4768         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4769         struct btrfs_ordered_extent *ordered;
4770         struct extent_state *cached_state = NULL;
4771         struct extent_changeset *data_reserved = NULL;
4772         char *kaddr;
4773         u32 blocksize = fs_info->sectorsize;
4774         pgoff_t index = from >> PAGE_SHIFT;
4775         unsigned offset = from & (blocksize - 1);
4776         struct page *page;
4777         gfp_t mask = btrfs_alloc_write_mask(mapping);
4778         int ret = 0;
4779         u64 block_start;
4780         u64 block_end;
4781
4782         if ((offset & (blocksize - 1)) == 0 &&
4783             (!len || ((len & (blocksize - 1)) == 0)))
4784                 goto out;
4785
4786         ret = btrfs_delalloc_reserve_space(inode, &data_reserved,
4787                         round_down(from, blocksize), blocksize);
4788         if (ret)
4789                 goto out;
4790
4791 again:
4792         page = find_or_create_page(mapping, index, mask);
4793         if (!page) {
4794                 btrfs_delalloc_release_space(inode, data_reserved,
4795                                 round_down(from, blocksize),
4796                                 blocksize);
4797                 ret = -ENOMEM;
4798                 goto out;
4799         }
4800
4801         block_start = round_down(from, blocksize);
4802         block_end = block_start + blocksize - 1;
4803
4804         if (!PageUptodate(page)) {
4805                 ret = btrfs_readpage(NULL, page);
4806                 lock_page(page);
4807                 if (page->mapping != mapping) {
4808                         unlock_page(page);
4809                         put_page(page);
4810                         goto again;
4811                 }
4812                 if (!PageUptodate(page)) {
4813                         ret = -EIO;
4814                         goto out_unlock;
4815                 }
4816         }
4817         wait_on_page_writeback(page);
4818
4819         lock_extent_bits(io_tree, block_start, block_end, &cached_state);
4820         set_page_extent_mapped(page);
4821
4822         ordered = btrfs_lookup_ordered_extent(inode, block_start);
4823         if (ordered) {
4824                 unlock_extent_cached(io_tree, block_start, block_end,
4825                                      &cached_state, GFP_NOFS);
4826                 unlock_page(page);
4827                 put_page(page);
4828                 btrfs_start_ordered_extent(inode, ordered, 1);
4829                 btrfs_put_ordered_extent(ordered);
4830                 goto again;
4831         }
4832
4833         clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end,
4834                           EXTENT_DIRTY | EXTENT_DELALLOC |
4835                           EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
4836                           0, 0, &cached_state, GFP_NOFS);
4837
4838         ret = btrfs_set_extent_delalloc(inode, block_start, block_end,
4839                                         &cached_state, 0);
4840         if (ret) {
4841                 unlock_extent_cached(io_tree, block_start, block_end,
4842                                      &cached_state, GFP_NOFS);
4843                 goto out_unlock;
4844         }
4845
4846         if (offset != blocksize) {
4847                 if (!len)
4848                         len = blocksize - offset;
4849                 kaddr = kmap(page);
4850                 if (front)
4851                         memset(kaddr + (block_start - page_offset(page)),
4852                                 0, offset);
4853                 else
4854                         memset(kaddr + (block_start - page_offset(page)) +  offset,
4855                                 0, len);
4856                 flush_dcache_page(page);
4857                 kunmap(page);
4858         }
4859         ClearPageChecked(page);
4860         set_page_dirty(page);
4861         unlock_extent_cached(io_tree, block_start, block_end, &cached_state,
4862                              GFP_NOFS);
4863
4864 out_unlock:
4865         if (ret)
4866                 btrfs_delalloc_release_space(inode, data_reserved, block_start,
4867                                              blocksize);
4868         unlock_page(page);
4869         put_page(page);
4870 out:
4871         extent_changeset_free(data_reserved);
4872         return ret;
4873 }
4874
4875 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode,
4876                              u64 offset, u64 len)
4877 {
4878         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4879         struct btrfs_trans_handle *trans;
4880         int ret;
4881
4882         /*
4883          * Still need to make sure the inode looks like it's been updated so
4884          * that any holes get logged if we fsync.
4885          */
4886         if (btrfs_fs_incompat(fs_info, NO_HOLES)) {
4887                 BTRFS_I(inode)->last_trans = fs_info->generation;
4888                 BTRFS_I(inode)->last_sub_trans = root->log_transid;
4889                 BTRFS_I(inode)->last_log_commit = root->last_log_commit;
4890                 return 0;
4891         }
4892
4893         /*
4894          * 1 - for the one we're dropping
4895          * 1 - for the one we're adding
4896          * 1 - for updating the inode.
4897          */
4898         trans = btrfs_start_transaction(root, 3);
4899         if (IS_ERR(trans))
4900                 return PTR_ERR(trans);
4901
4902         ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1);
4903         if (ret) {
4904                 btrfs_abort_transaction(trans, ret);
4905                 btrfs_end_transaction(trans);
4906                 return ret;
4907         }
4908
4909         ret = btrfs_insert_file_extent(trans, root, btrfs_ino(BTRFS_I(inode)),
4910                         offset, 0, 0, len, 0, len, 0, 0, 0);
4911         if (ret)
4912                 btrfs_abort_transaction(trans, ret);
4913         else
4914                 btrfs_update_inode(trans, root, inode);
4915         btrfs_end_transaction(trans);
4916         return ret;
4917 }
4918
4919 /*
4920  * This function puts in dummy file extents for the area we're creating a hole
4921  * for.  So if we are truncating this file to a larger size we need to insert
4922  * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
4923  * the range between oldsize and size
4924  */
4925 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size)
4926 {
4927         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4928         struct btrfs_root *root = BTRFS_I(inode)->root;
4929         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4930         struct extent_map *em = NULL;
4931         struct extent_state *cached_state = NULL;
4932         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
4933         u64 hole_start = ALIGN(oldsize, fs_info->sectorsize);
4934         u64 block_end = ALIGN(size, fs_info->sectorsize);
4935         u64 last_byte;
4936         u64 cur_offset;
4937         u64 hole_size;
4938         int err = 0;
4939
4940         /*
4941          * If our size started in the middle of a block we need to zero out the
4942          * rest of the block before we expand the i_size, otherwise we could
4943          * expose stale data.
4944          */
4945         err = btrfs_truncate_block(inode, oldsize, 0, 0);
4946         if (err)
4947                 return err;
4948
4949         if (size <= hole_start)
4950                 return 0;
4951
4952         while (1) {
4953                 struct btrfs_ordered_extent *ordered;
4954
4955                 lock_extent_bits(io_tree, hole_start, block_end - 1,
4956                                  &cached_state);
4957                 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), hole_start,
4958                                                      block_end - hole_start);
4959                 if (!ordered)
4960                         break;
4961                 unlock_extent_cached(io_tree, hole_start, block_end - 1,
4962                                      &cached_state, GFP_NOFS);
4963                 btrfs_start_ordered_extent(inode, ordered, 1);
4964                 btrfs_put_ordered_extent(ordered);
4965         }
4966
4967         cur_offset = hole_start;
4968         while (1) {
4969                 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
4970                                 block_end - cur_offset, 0);
4971                 if (IS_ERR(em)) {
4972                         err = PTR_ERR(em);
4973                         em = NULL;
4974                         break;
4975                 }
4976                 last_byte = min(extent_map_end(em), block_end);
4977                 last_byte = ALIGN(last_byte, fs_info->sectorsize);
4978                 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
4979                         struct extent_map *hole_em;
4980                         hole_size = last_byte - cur_offset;
4981
4982                         err = maybe_insert_hole(root, inode, cur_offset,
4983                                                 hole_size);
4984                         if (err)
4985                                 break;
4986                         btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
4987                                                 cur_offset + hole_size - 1, 0);
4988                         hole_em = alloc_extent_map();
4989                         if (!hole_em) {
4990                                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
4991                                         &BTRFS_I(inode)->runtime_flags);
4992                                 goto next;
4993                         }
4994                         hole_em->start = cur_offset;
4995                         hole_em->len = hole_size;
4996                         hole_em->orig_start = cur_offset;
4997
4998                         hole_em->block_start = EXTENT_MAP_HOLE;
4999                         hole_em->block_len = 0;
5000                         hole_em->orig_block_len = 0;
5001                         hole_em->ram_bytes = hole_size;
5002                         hole_em->bdev = fs_info->fs_devices->latest_bdev;
5003                         hole_em->compress_type = BTRFS_COMPRESS_NONE;
5004                         hole_em->generation = fs_info->generation;
5005
5006                         while (1) {
5007                                 write_lock(&em_tree->lock);
5008                                 err = add_extent_mapping(em_tree, hole_em, 1);
5009                                 write_unlock(&em_tree->lock);
5010                                 if (err != -EEXIST)
5011                                         break;
5012                                 btrfs_drop_extent_cache(BTRFS_I(inode),
5013                                                         cur_offset,
5014                                                         cur_offset +
5015                                                         hole_size - 1, 0);
5016                         }
5017                         free_extent_map(hole_em);
5018                 }
5019 next:
5020                 free_extent_map(em);
5021                 em = NULL;
5022                 cur_offset = last_byte;
5023                 if (cur_offset >= block_end)
5024                         break;
5025         }
5026         free_extent_map(em);
5027         unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state,
5028                              GFP_NOFS);
5029         return err;
5030 }
5031
5032 static int btrfs_setsize(struct inode *inode, struct iattr *attr)
5033 {
5034         struct btrfs_root *root = BTRFS_I(inode)->root;
5035         struct btrfs_trans_handle *trans;
5036         loff_t oldsize = i_size_read(inode);
5037         loff_t newsize = attr->ia_size;
5038         int mask = attr->ia_valid;
5039         int ret;
5040
5041         /*
5042          * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
5043          * special case where we need to update the times despite not having
5044          * these flags set.  For all other operations the VFS set these flags
5045          * explicitly if it wants a timestamp update.
5046          */
5047         if (newsize != oldsize) {
5048                 inode_inc_iversion(inode);
5049                 if (!(mask & (ATTR_CTIME | ATTR_MTIME)))
5050                         inode->i_ctime = inode->i_mtime =
5051                                 current_time(inode);
5052         }
5053
5054         if (newsize > oldsize) {
5055                 /*
5056                  * Don't do an expanding truncate while snapshoting is ongoing.
5057                  * This is to ensure the snapshot captures a fully consistent
5058                  * state of this file - if the snapshot captures this expanding
5059                  * truncation, it must capture all writes that happened before
5060                  * this truncation.
5061                  */
5062                 btrfs_wait_for_snapshot_creation(root);
5063                 ret = btrfs_cont_expand(inode, oldsize, newsize);
5064                 if (ret) {
5065                         btrfs_end_write_no_snapshoting(root);
5066                         return ret;
5067                 }
5068
5069                 trans = btrfs_start_transaction(root, 1);
5070                 if (IS_ERR(trans)) {
5071                         btrfs_end_write_no_snapshoting(root);
5072                         return PTR_ERR(trans);
5073                 }
5074
5075                 i_size_write(inode, newsize);
5076                 btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL);
5077                 pagecache_isize_extended(inode, oldsize, newsize);
5078                 ret = btrfs_update_inode(trans, root, inode);
5079                 btrfs_end_write_no_snapshoting(root);
5080                 btrfs_end_transaction(trans);
5081         } else {
5082
5083                 /*
5084                  * We're truncating a file that used to have good data down to
5085                  * zero. Make sure it gets into the ordered flush list so that
5086                  * any new writes get down to disk quickly.
5087                  */
5088                 if (newsize == 0)
5089                         set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
5090                                 &BTRFS_I(inode)->runtime_flags);
5091
5092                 /*
5093                  * 1 for the orphan item we're going to add
5094                  * 1 for the orphan item deletion.
5095                  */
5096                 trans = btrfs_start_transaction(root, 2);
5097                 if (IS_ERR(trans))
5098                         return PTR_ERR(trans);
5099
5100                 /*
5101                  * We need to do this in case we fail at _any_ point during the
5102                  * actual truncate.  Once we do the truncate_setsize we could
5103                  * invalidate pages which forces any outstanding ordered io to
5104                  * be instantly completed which will give us extents that need
5105                  * to be truncated.  If we fail to get an orphan inode down we
5106                  * could have left over extents that were never meant to live,
5107                  * so we need to guarantee from this point on that everything
5108                  * will be consistent.
5109                  */
5110                 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
5111                 btrfs_end_transaction(trans);
5112                 if (ret)
5113                         return ret;
5114
5115                 /* we don't support swapfiles, so vmtruncate shouldn't fail */
5116                 truncate_setsize(inode, newsize);
5117
5118                 /* Disable nonlocked read DIO to avoid the end less truncate */
5119                 btrfs_inode_block_unlocked_dio(BTRFS_I(inode));
5120                 inode_dio_wait(inode);
5121                 btrfs_inode_resume_unlocked_dio(BTRFS_I(inode));
5122
5123                 ret = btrfs_truncate(inode);
5124                 if (ret && inode->i_nlink) {
5125                         int err;
5126
5127                         /* To get a stable disk_i_size */
5128                         err = btrfs_wait_ordered_range(inode, 0, (u64)-1);
5129                         if (err) {
5130                                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5131                                 return err;
5132                         }
5133
5134                         /*
5135                          * failed to truncate, disk_i_size is only adjusted down
5136                          * as we remove extents, so it should represent the true
5137                          * size of the inode, so reset the in memory size and
5138                          * delete our orphan entry.
5139                          */
5140                         trans = btrfs_join_transaction(root);
5141                         if (IS_ERR(trans)) {
5142                                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5143                                 return ret;
5144                         }
5145                         i_size_write(inode, BTRFS_I(inode)->disk_i_size);
5146                         err = btrfs_orphan_del(trans, BTRFS_I(inode));
5147                         if (err)
5148                                 btrfs_abort_transaction(trans, err);
5149                         btrfs_end_transaction(trans);
5150                 }
5151         }
5152
5153         return ret;
5154 }
5155
5156 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr)
5157 {
5158         struct inode *inode = d_inode(dentry);
5159         struct btrfs_root *root = BTRFS_I(inode)->root;
5160         int err;
5161
5162         if (btrfs_root_readonly(root))
5163                 return -EROFS;
5164
5165         err = setattr_prepare(dentry, attr);
5166         if (err)
5167                 return err;
5168
5169         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
5170                 err = btrfs_setsize(inode, attr);
5171                 if (err)
5172                         return err;
5173         }
5174
5175         if (attr->ia_valid) {
5176                 setattr_copy(inode, attr);
5177                 inode_inc_iversion(inode);
5178                 err = btrfs_dirty_inode(inode);
5179
5180                 if (!err && attr->ia_valid & ATTR_MODE)
5181                         err = posix_acl_chmod(inode, inode->i_mode);
5182         }
5183
5184         return err;
5185 }
5186
5187 /*
5188  * While truncating the inode pages during eviction, we get the VFS calling
5189  * btrfs_invalidatepage() against each page of the inode. This is slow because
5190  * the calls to btrfs_invalidatepage() result in a huge amount of calls to
5191  * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
5192  * extent_state structures over and over, wasting lots of time.
5193  *
5194  * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
5195  * those expensive operations on a per page basis and do only the ordered io
5196  * finishing, while we release here the extent_map and extent_state structures,
5197  * without the excessive merging and splitting.
5198  */
5199 static void evict_inode_truncate_pages(struct inode *inode)
5200 {
5201         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
5202         struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree;
5203         struct rb_node *node;
5204
5205         ASSERT(inode->i_state & I_FREEING);
5206         truncate_inode_pages_final(&inode->i_data);
5207
5208         write_lock(&map_tree->lock);
5209         while (!RB_EMPTY_ROOT(&map_tree->map)) {
5210                 struct extent_map *em;
5211
5212                 node = rb_first(&map_tree->map);
5213                 em = rb_entry(node, struct extent_map, rb_node);
5214                 clear_bit(EXTENT_FLAG_PINNED, &em->flags);
5215                 clear_bit(EXTENT_FLAG_LOGGING, &em->flags);
5216                 remove_extent_mapping(map_tree, em);
5217                 free_extent_map(em);
5218                 if (need_resched()) {
5219                         write_unlock(&map_tree->lock);
5220                         cond_resched();
5221                         write_lock(&map_tree->lock);
5222                 }
5223         }
5224         write_unlock(&map_tree->lock);
5225
5226         /*
5227          * Keep looping until we have no more ranges in the io tree.
5228          * We can have ongoing bios started by readpages (called from readahead)
5229          * that have their endio callback (extent_io.c:end_bio_extent_readpage)
5230          * still in progress (unlocked the pages in the bio but did not yet
5231          * unlocked the ranges in the io tree). Therefore this means some
5232          * ranges can still be locked and eviction started because before
5233          * submitting those bios, which are executed by a separate task (work
5234          * queue kthread), inode references (inode->i_count) were not taken
5235          * (which would be dropped in the end io callback of each bio).
5236          * Therefore here we effectively end up waiting for those bios and
5237          * anyone else holding locked ranges without having bumped the inode's
5238          * reference count - if we don't do it, when they access the inode's
5239          * io_tree to unlock a range it may be too late, leading to an
5240          * use-after-free issue.
5241          */
5242         spin_lock(&io_tree->lock);
5243         while (!RB_EMPTY_ROOT(&io_tree->state)) {
5244                 struct extent_state *state;
5245                 struct extent_state *cached_state = NULL;
5246                 u64 start;
5247                 u64 end;
5248
5249                 node = rb_first(&io_tree->state);
5250                 state = rb_entry(node, struct extent_state, rb_node);
5251                 start = state->start;
5252                 end = state->end;
5253                 spin_unlock(&io_tree->lock);
5254
5255                 lock_extent_bits(io_tree, start, end, &cached_state);
5256
5257                 /*
5258                  * If still has DELALLOC flag, the extent didn't reach disk,
5259                  * and its reserved space won't be freed by delayed_ref.
5260                  * So we need to free its reserved space here.
5261                  * (Refer to comment in btrfs_invalidatepage, case 2)
5262                  *
5263                  * Note, end is the bytenr of last byte, so we need + 1 here.
5264                  */
5265                 if (state->state & EXTENT_DELALLOC)
5266                         btrfs_qgroup_free_data(inode, NULL, start, end - start + 1);
5267
5268                 clear_extent_bit(io_tree, start, end,
5269                                  EXTENT_LOCKED | EXTENT_DIRTY |
5270                                  EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
5271                                  EXTENT_DEFRAG, 1, 1,
5272                                  &cached_state, GFP_NOFS);
5273
5274                 cond_resched();
5275                 spin_lock(&io_tree->lock);
5276         }
5277         spin_unlock(&io_tree->lock);
5278 }
5279
5280 void btrfs_evict_inode(struct inode *inode)
5281 {
5282         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5283         struct btrfs_trans_handle *trans;
5284         struct btrfs_root *root = BTRFS_I(inode)->root;
5285         struct btrfs_block_rsv *rsv, *global_rsv;
5286         int steal_from_global = 0;
5287         u64 min_size;
5288         int ret;
5289
5290         trace_btrfs_inode_evict(inode);
5291
5292         if (!root) {
5293                 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
5294                 return;
5295         }
5296
5297         min_size = btrfs_calc_trunc_metadata_size(fs_info, 1);
5298
5299         evict_inode_truncate_pages(inode);
5300
5301         if (inode->i_nlink &&
5302             ((btrfs_root_refs(&root->root_item) != 0 &&
5303               root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) ||
5304              btrfs_is_free_space_inode(BTRFS_I(inode))))
5305                 goto no_delete;
5306
5307         if (is_bad_inode(inode)) {
5308                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5309                 goto no_delete;
5310         }
5311         /* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */
5312         if (!special_file(inode->i_mode))
5313                 btrfs_wait_ordered_range(inode, 0, (u64)-1);
5314
5315         btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1);
5316
5317         if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
5318                 BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
5319                                  &BTRFS_I(inode)->runtime_flags));
5320                 goto no_delete;
5321         }
5322
5323         if (inode->i_nlink > 0) {
5324                 BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
5325                        root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID);
5326                 goto no_delete;
5327         }
5328
5329         ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));
5330         if (ret) {
5331                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5332                 goto no_delete;
5333         }
5334
5335         rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
5336         if (!rsv) {
5337                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5338                 goto no_delete;
5339         }
5340         rsv->size = min_size;
5341         rsv->failfast = 1;
5342         global_rsv = &fs_info->global_block_rsv;
5343
5344         btrfs_i_size_write(BTRFS_I(inode), 0);
5345
5346         /*
5347          * This is a bit simpler than btrfs_truncate since we've already
5348          * reserved our space for our orphan item in the unlink, so we just
5349          * need to reserve some slack space in case we add bytes and update
5350          * inode item when doing the truncate.
5351          */
5352         while (1) {
5353                 ret = btrfs_block_rsv_refill(root, rsv, min_size,
5354                                              BTRFS_RESERVE_FLUSH_LIMIT);
5355
5356                 /*
5357                  * Try and steal from the global reserve since we will
5358                  * likely not use this space anyway, we want to try as
5359                  * hard as possible to get this to work.
5360                  */
5361                 if (ret)
5362                         steal_from_global++;
5363                 else
5364                         steal_from_global = 0;
5365                 ret = 0;
5366
5367                 /*
5368                  * steal_from_global == 0: we reserved stuff, hooray!
5369                  * steal_from_global == 1: we didn't reserve stuff, boo!
5370                  * steal_from_global == 2: we've committed, still not a lot of
5371                  * room but maybe we'll have room in the global reserve this
5372                  * time.
5373                  * steal_from_global == 3: abandon all hope!
5374                  */
5375                 if (steal_from_global > 2) {
5376                         btrfs_warn(fs_info,
5377                                    "Could not get space for a delete, will truncate on mount %d",
5378                                    ret);
5379                         btrfs_orphan_del(NULL, BTRFS_I(inode));
5380                         btrfs_free_block_rsv(fs_info, rsv);
5381                         goto no_delete;
5382                 }
5383
5384                 trans = btrfs_join_transaction(root);
5385                 if (IS_ERR(trans)) {
5386                         btrfs_orphan_del(NULL, BTRFS_I(inode));
5387                         btrfs_free_block_rsv(fs_info, rsv);
5388                         goto no_delete;
5389                 }
5390
5391                 /*
5392                  * We can't just steal from the global reserve, we need to make
5393                  * sure there is room to do it, if not we need to commit and try
5394                  * again.
5395                  */
5396                 if (steal_from_global) {
5397                         if (!btrfs_check_space_for_delayed_refs(trans, fs_info))
5398                                 ret = btrfs_block_rsv_migrate(global_rsv, rsv,
5399                                                               min_size, 0);
5400                         else
5401                                 ret = -ENOSPC;
5402                 }
5403
5404                 /*
5405                  * Couldn't steal from the global reserve, we have too much
5406                  * pending stuff built up, commit the transaction and try it
5407                  * again.
5408                  */
5409                 if (ret) {
5410                         ret = btrfs_commit_transaction(trans);
5411                         if (ret) {
5412                                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5413                                 btrfs_free_block_rsv(fs_info, rsv);
5414                                 goto no_delete;
5415                         }
5416                         continue;
5417                 } else {
5418                         steal_from_global = 0;
5419                 }
5420
5421                 trans->block_rsv = rsv;
5422
5423                 ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0);
5424                 if (ret != -ENOSPC && ret != -EAGAIN)
5425                         break;
5426
5427                 trans->block_rsv = &fs_info->trans_block_rsv;
5428                 btrfs_end_transaction(trans);
5429                 trans = NULL;
5430                 btrfs_btree_balance_dirty(fs_info);
5431         }
5432
5433         btrfs_free_block_rsv(fs_info, rsv);
5434
5435         /*
5436          * Errors here aren't a big deal, it just means we leave orphan items
5437          * in the tree.  They will be cleaned up on the next mount.
5438          */
5439         if (ret == 0) {
5440                 trans->block_rsv = root->orphan_block_rsv;
5441                 btrfs_orphan_del(trans, BTRFS_I(inode));
5442         } else {
5443                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5444         }
5445
5446         trans->block_rsv = &fs_info->trans_block_rsv;
5447         if (!(root == fs_info->tree_root ||
5448               root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID))
5449                 btrfs_return_ino(root, btrfs_ino(BTRFS_I(inode)));
5450
5451         btrfs_end_transaction(trans);
5452         btrfs_btree_balance_dirty(fs_info);
5453 no_delete:
5454         btrfs_remove_delayed_node(BTRFS_I(inode));
5455         clear_inode(inode);
5456 }
5457
5458 /*
5459  * this returns the key found in the dir entry in the location pointer.
5460  * If no dir entries were found, location->objectid is 0.
5461  */
5462 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry,
5463                                struct btrfs_key *location)
5464 {
5465         const char *name = dentry->d_name.name;
5466         int namelen = dentry->d_name.len;
5467         struct btrfs_dir_item *di;
5468         struct btrfs_path *path;
5469         struct btrfs_root *root = BTRFS_I(dir)->root;
5470         int ret = 0;
5471
5472         path = btrfs_alloc_path();
5473         if (!path)
5474                 return -ENOMEM;
5475
5476         di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(BTRFS_I(dir)),
5477                         name, namelen, 0);
5478         if (IS_ERR(di))
5479                 ret = PTR_ERR(di);
5480
5481         if (IS_ERR_OR_NULL(di))
5482                 goto out_err;
5483
5484         btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);
5485 out:
5486         btrfs_free_path(path);
5487         return ret;
5488 out_err:
5489         location->objectid = 0;
5490         goto out;
5491 }
5492
5493 /*
5494  * when we hit a tree root in a directory, the btrfs part of the inode
5495  * needs to be changed to reflect the root directory of the tree root.  This
5496  * is kind of like crossing a mount point.
5497  */
5498 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info,
5499                                     struct inode *dir,
5500                                     struct dentry *dentry,
5501                                     struct btrfs_key *location,
5502                                     struct btrfs_root **sub_root)
5503 {
5504         struct btrfs_path *path;
5505         struct btrfs_root *new_root;
5506         struct btrfs_root_ref *ref;
5507         struct extent_buffer *leaf;
5508         struct btrfs_key key;
5509         int ret;
5510         int err = 0;
5511
5512         path = btrfs_alloc_path();
5513         if (!path) {
5514                 err = -ENOMEM;
5515                 goto out;
5516         }
5517
5518         err = -ENOENT;
5519         key.objectid = BTRFS_I(dir)->root->root_key.objectid;
5520         key.type = BTRFS_ROOT_REF_KEY;
5521         key.offset = location->objectid;
5522
5523         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
5524         if (ret) {
5525                 if (ret < 0)
5526                         err = ret;
5527                 goto out;
5528         }
5529
5530         leaf = path->nodes[0];
5531         ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
5532         if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(BTRFS_I(dir)) ||
5533             btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len)
5534                 goto out;
5535
5536         ret = memcmp_extent_buffer(leaf, dentry->d_name.name,
5537                                    (unsigned long)(ref + 1),
5538                                    dentry->d_name.len);
5539         if (ret)
5540                 goto out;
5541
5542         btrfs_release_path(path);
5543
5544         new_root = btrfs_read_fs_root_no_name(fs_info, location);
5545         if (IS_ERR(new_root)) {
5546                 err = PTR_ERR(new_root);
5547                 goto out;
5548         }
5549
5550         *sub_root = new_root;
5551         location->objectid = btrfs_root_dirid(&new_root->root_item);
5552         location->type = BTRFS_INODE_ITEM_KEY;
5553         location->offset = 0;
5554         err = 0;
5555 out:
5556         btrfs_free_path(path);
5557         return err;
5558 }
5559
5560 static void inode_tree_add(struct inode *inode)
5561 {
5562         struct btrfs_root *root = BTRFS_I(inode)->root;
5563         struct btrfs_inode *entry;
5564         struct rb_node **p;
5565         struct rb_node *parent;
5566         struct rb_node *new = &BTRFS_I(inode)->rb_node;
5567         u64 ino = btrfs_ino(BTRFS_I(inode));
5568
5569         if (inode_unhashed(inode))
5570                 return;
5571         parent = NULL;
5572         spin_lock(&root->inode_lock);
5573         p = &root->inode_tree.rb_node;
5574         while (*p) {
5575                 parent = *p;
5576                 entry = rb_entry(parent, struct btrfs_inode, rb_node);
5577
5578                 if (ino < btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5579                         p = &parent->rb_left;
5580                 else if (ino > btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5581                         p = &parent->rb_right;
5582                 else {
5583                         WARN_ON(!(entry->vfs_inode.i_state &
5584                                   (I_WILL_FREE | I_FREEING)));
5585                         rb_replace_node(parent, new, &root->inode_tree);
5586                         RB_CLEAR_NODE(parent);
5587                         spin_unlock(&root->inode_lock);
5588                         return;
5589                 }
5590         }
5591         rb_link_node(new, parent, p);
5592         rb_insert_color(new, &root->inode_tree);
5593         spin_unlock(&root->inode_lock);
5594 }
5595
5596 static void inode_tree_del(struct inode *inode)
5597 {
5598         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5599         struct btrfs_root *root = BTRFS_I(inode)->root;
5600         int empty = 0;
5601
5602         spin_lock(&root->inode_lock);
5603         if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) {
5604                 rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree);
5605                 RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
5606                 empty = RB_EMPTY_ROOT(&root->inode_tree);
5607         }
5608         spin_unlock(&root->inode_lock);
5609
5610         if (empty && btrfs_root_refs(&root->root_item) == 0) {
5611                 synchronize_srcu(&fs_info->subvol_srcu);
5612                 spin_lock(&root->inode_lock);
5613                 empty = RB_EMPTY_ROOT(&root->inode_tree);
5614                 spin_unlock(&root->inode_lock);
5615                 if (empty)
5616                         btrfs_add_dead_root(root);
5617         }
5618 }
5619
5620 void btrfs_invalidate_inodes(struct btrfs_root *root)
5621 {
5622         struct btrfs_fs_info *fs_info = root->fs_info;
5623         struct rb_node *node;
5624         struct rb_node *prev;
5625         struct btrfs_inode *entry;
5626         struct inode *inode;
5627         u64 objectid = 0;
5628
5629         if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
5630                 WARN_ON(btrfs_root_refs(&root->root_item) != 0);
5631
5632         spin_lock(&root->inode_lock);
5633 again:
5634         node = root->inode_tree.rb_node;
5635         prev = NULL;
5636         while (node) {
5637                 prev = node;
5638                 entry = rb_entry(node, struct btrfs_inode, rb_node);
5639
5640                 if (objectid < btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5641                         node = node->rb_left;
5642                 else if (objectid > btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5643                         node = node->rb_right;
5644                 else
5645                         break;
5646         }
5647         if (!node) {
5648                 while (prev) {
5649                         entry = rb_entry(prev, struct btrfs_inode, rb_node);
5650                         if (objectid <= btrfs_ino(BTRFS_I(&entry->vfs_inode))) {
5651                                 node = prev;
5652                                 break;
5653                         }
5654                         prev = rb_next(prev);
5655                 }
5656         }
5657         while (node) {
5658                 entry = rb_entry(node, struct btrfs_inode, rb_node);
5659                 objectid = btrfs_ino(BTRFS_I(&entry->vfs_inode)) + 1;
5660                 inode = igrab(&entry->vfs_inode);
5661                 if (inode) {
5662                         spin_unlock(&root->inode_lock);
5663                         if (atomic_read(&inode->i_count) > 1)
5664                                 d_prune_aliases(inode);
5665                         /*
5666                          * btrfs_drop_inode will have it removed from
5667                          * the inode cache when its usage count
5668                          * hits zero.
5669                          */
5670                         iput(inode);
5671                         cond_resched();
5672                         spin_lock(&root->inode_lock);
5673                         goto again;
5674                 }
5675
5676                 if (cond_resched_lock(&root->inode_lock))
5677                         goto again;
5678
5679                 node = rb_next(node);
5680         }
5681         spin_unlock(&root->inode_lock);
5682 }
5683
5684 static int btrfs_init_locked_inode(struct inode *inode, void *p)
5685 {
5686         struct btrfs_iget_args *args = p;
5687         inode->i_ino = args->location->objectid;
5688         memcpy(&BTRFS_I(inode)->location, args->location,
5689                sizeof(*args->location));
5690         BTRFS_I(inode)->root = args->root;
5691         return 0;
5692 }
5693
5694 static int btrfs_find_actor(struct inode *inode, void *opaque)
5695 {
5696         struct btrfs_iget_args *args = opaque;
5697         return args->location->objectid == BTRFS_I(inode)->location.objectid &&
5698                 args->root == BTRFS_I(inode)->root;
5699 }
5700
5701 static struct inode *btrfs_iget_locked(struct super_block *s,
5702                                        struct btrfs_key *location,
5703                                        struct btrfs_root *root)
5704 {
5705         struct inode *inode;
5706         struct btrfs_iget_args args;
5707         unsigned long hashval = btrfs_inode_hash(location->objectid, root);
5708
5709         args.location = location;
5710         args.root = root;
5711
5712         inode = iget5_locked(s, hashval, btrfs_find_actor,
5713                              btrfs_init_locked_inode,
5714                              (void *)&args);
5715         return inode;
5716 }
5717
5718 /* Get an inode object given its location and corresponding root.
5719  * Returns in *is_new if the inode was read from disk
5720  */
5721 struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location,
5722                          struct btrfs_root *root, int *new)
5723 {
5724         struct inode *inode;
5725
5726         inode = btrfs_iget_locked(s, location, root);
5727         if (!inode)
5728                 return ERR_PTR(-ENOMEM);
5729
5730         if (inode->i_state & I_NEW) {
5731                 int ret;
5732
5733                 ret = btrfs_read_locked_inode(inode);
5734                 if (!is_bad_inode(inode)) {
5735                         inode_tree_add(inode);
5736                         unlock_new_inode(inode);
5737                         if (new)
5738                                 *new = 1;
5739                 } else {
5740                         unlock_new_inode(inode);
5741                         iput(inode);
5742                         ASSERT(ret < 0);
5743                         inode = ERR_PTR(ret < 0 ? ret : -ESTALE);
5744                 }
5745         }
5746
5747         return inode;
5748 }
5749
5750 static struct inode *new_simple_dir(struct super_block *s,
5751                                     struct btrfs_key *key,
5752                                     struct btrfs_root *root)
5753 {
5754         struct inode *inode = new_inode(s);
5755
5756         if (!inode)
5757                 return ERR_PTR(-ENOMEM);
5758
5759         BTRFS_I(inode)->root = root;
5760         memcpy(&BTRFS_I(inode)->location, key, sizeof(*key));
5761         set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
5762
5763         inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID;
5764         inode->i_op = &btrfs_dir_ro_inode_operations;
5765         inode->i_opflags &= ~IOP_XATTR;
5766         inode->i_fop = &simple_dir_operations;
5767         inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;
5768         inode->i_mtime = current_time(inode);
5769         inode->i_atime = inode->i_mtime;
5770         inode->i_ctime = inode->i_mtime;
5771         BTRFS_I(inode)->i_otime = inode->i_mtime;
5772
5773         return inode;
5774 }
5775
5776 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)
5777 {
5778         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
5779         struct inode *inode;
5780         struct btrfs_root *root = BTRFS_I(dir)->root;
5781         struct btrfs_root *sub_root = root;
5782         struct btrfs_key location;
5783         int index;
5784         int ret = 0;
5785
5786         if (dentry->d_name.len > BTRFS_NAME_LEN)
5787                 return ERR_PTR(-ENAMETOOLONG);
5788
5789         ret = btrfs_inode_by_name(dir, dentry, &location);
5790         if (ret < 0)
5791                 return ERR_PTR(ret);
5792
5793         if (location.objectid == 0)
5794                 return ERR_PTR(-ENOENT);
5795
5796         if (location.type == BTRFS_INODE_ITEM_KEY) {
5797                 inode = btrfs_iget(dir->i_sb, &location, root, NULL);
5798                 return inode;
5799         }
5800
5801         BUG_ON(location.type != BTRFS_ROOT_ITEM_KEY);
5802
5803         index = srcu_read_lock(&fs_info->subvol_srcu);
5804         ret = fixup_tree_root_location(fs_info, dir, dentry,
5805                                        &location, &sub_root);
5806         if (ret < 0) {
5807                 if (ret != -ENOENT)
5808                         inode = ERR_PTR(ret);
5809                 else
5810                         inode = new_simple_dir(dir->i_sb, &location, sub_root);
5811         } else {
5812                 inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL);
5813         }
5814         srcu_read_unlock(&fs_info->subvol_srcu, index);
5815
5816         if (!IS_ERR(inode) && root != sub_root) {
5817                 down_read(&fs_info->cleanup_work_sem);
5818                 if (!(inode->i_sb->s_flags & MS_RDONLY))
5819                         ret = btrfs_orphan_cleanup(sub_root);
5820                 up_read(&fs_info->cleanup_work_sem);
5821                 if (ret) {
5822                         iput(inode);
5823                         inode = ERR_PTR(ret);
5824                 }
5825         }
5826
5827         return inode;
5828 }
5829
5830 static int btrfs_dentry_delete(const struct dentry *dentry)
5831 {
5832         struct btrfs_root *root;
5833         struct inode *inode = d_inode(dentry);
5834
5835         if (!inode && !IS_ROOT(dentry))
5836                 inode = d_inode(dentry->d_parent);
5837
5838         if (inode) {
5839                 root = BTRFS_I(inode)->root;
5840                 if (btrfs_root_refs(&root->root_item) == 0)
5841                         return 1;
5842
5843                 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
5844                         return 1;
5845         }
5846         return 0;
5847 }
5848
5849 static void btrfs_dentry_release(struct dentry *dentry)
5850 {
5851         kfree(dentry->d_fsdata);
5852 }
5853
5854 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,
5855                                    unsigned int flags)
5856 {
5857         struct inode *inode;
5858
5859         inode = btrfs_lookup_dentry(dir, dentry);
5860         if (IS_ERR(inode)) {
5861                 if (PTR_ERR(inode) == -ENOENT)
5862                         inode = NULL;
5863                 else
5864                         return ERR_CAST(inode);
5865         }
5866
5867         return d_splice_alias(inode, dentry);
5868 }
5869
5870 unsigned char btrfs_filetype_table[] = {
5871         DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK
5872 };
5873
5874 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
5875 {
5876         struct inode *inode = file_inode(file);
5877         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5878         struct btrfs_root *root = BTRFS_I(inode)->root;
5879         struct btrfs_dir_item *di;
5880         struct btrfs_key key;
5881         struct btrfs_key found_key;
5882         struct btrfs_path *path;
5883         struct list_head ins_list;
5884         struct list_head del_list;
5885         int ret;
5886         struct extent_buffer *leaf;
5887         int slot;
5888         unsigned char d_type;
5889         int over = 0;
5890         char tmp_name[32];
5891         char *name_ptr;
5892         int name_len;
5893         bool put = false;
5894         struct btrfs_key location;
5895
5896         if (!dir_emit_dots(file, ctx))
5897                 return 0;
5898
5899         path = btrfs_alloc_path();
5900         if (!path)
5901                 return -ENOMEM;
5902
5903         path->reada = READA_FORWARD;
5904
5905         INIT_LIST_HEAD(&ins_list);
5906         INIT_LIST_HEAD(&del_list);
5907         put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list);
5908
5909         key.type = BTRFS_DIR_INDEX_KEY;
5910         key.offset = ctx->pos;
5911         key.objectid = btrfs_ino(BTRFS_I(inode));
5912
5913         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5914         if (ret < 0)
5915                 goto err;
5916
5917         while (1) {
5918                 leaf = path->nodes[0];
5919                 slot = path->slots[0];
5920                 if (slot >= btrfs_header_nritems(leaf)) {
5921                         ret = btrfs_next_leaf(root, path);
5922                         if (ret < 0)
5923                                 goto err;
5924                         else if (ret > 0)
5925                                 break;
5926                         continue;
5927                 }
5928
5929                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
5930
5931                 if (found_key.objectid != key.objectid)
5932                         break;
5933                 if (found_key.type != BTRFS_DIR_INDEX_KEY)
5934                         break;
5935                 if (found_key.offset < ctx->pos)
5936                         goto next;
5937                 if (btrfs_should_delete_dir_index(&del_list, found_key.offset))
5938                         goto next;
5939
5940                 ctx->pos = found_key.offset;
5941
5942                 di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item);
5943                 if (verify_dir_item(fs_info, leaf, slot, di))
5944                         goto next;
5945
5946                 name_len = btrfs_dir_name_len(leaf, di);
5947                 if (name_len <= sizeof(tmp_name)) {
5948                         name_ptr = tmp_name;
5949                 } else {
5950                         name_ptr = kmalloc(name_len, GFP_KERNEL);
5951                         if (!name_ptr) {
5952                                 ret = -ENOMEM;
5953                                 goto err;
5954                         }
5955                 }
5956                 read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1),
5957                                    name_len);
5958
5959                 d_type = btrfs_filetype_table[btrfs_dir_type(leaf, di)];
5960                 btrfs_dir_item_key_to_cpu(leaf, di, &location);
5961
5962                 over = !dir_emit(ctx, name_ptr, name_len, location.objectid,
5963                                  d_type);
5964
5965                 if (name_ptr != tmp_name)
5966                         kfree(name_ptr);
5967
5968                 if (over)
5969                         goto nopos;
5970                 ctx->pos++;
5971 next:
5972                 path->slots[0]++;
5973         }
5974
5975         ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list);
5976         if (ret)
5977                 goto nopos;
5978
5979         /*
5980          * Stop new entries from being returned after we return the last
5981          * entry.
5982          *
5983          * New directory entries are assigned a strictly increasing
5984          * offset.  This means that new entries created during readdir
5985          * are *guaranteed* to be seen in the future by that readdir.
5986          * This has broken buggy programs which operate on names as
5987          * they're returned by readdir.  Until we re-use freed offsets
5988          * we have this hack to stop new entries from being returned
5989          * under the assumption that they'll never reach this huge
5990          * offset.
5991          *
5992          * This is being careful not to overflow 32bit loff_t unless the
5993          * last entry requires it because doing so has broken 32bit apps
5994          * in the past.
5995          */
5996         if (ctx->pos >= INT_MAX)
5997                 ctx->pos = LLONG_MAX;
5998         else
5999                 ctx->pos = INT_MAX;
6000 nopos:
6001         ret = 0;
6002 err:
6003         if (put)
6004                 btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list);
6005         btrfs_free_path(path);
6006         return ret;
6007 }
6008
6009 int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc)
6010 {
6011         struct btrfs_root *root = BTRFS_I(inode)->root;
6012         struct btrfs_trans_handle *trans;
6013         int ret = 0;
6014         bool nolock = false;
6015
6016         if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
6017                 return 0;
6018
6019         if (btrfs_fs_closing(root->fs_info) &&
6020                         btrfs_is_free_space_inode(BTRFS_I(inode)))
6021                 nolock = true;
6022
6023         if (wbc->sync_mode == WB_SYNC_ALL) {
6024                 if (nolock)
6025                         trans = btrfs_join_transaction_nolock(root);
6026                 else
6027                         trans = btrfs_join_transaction(root);
6028                 if (IS_ERR(trans))
6029                         return PTR_ERR(trans);
6030                 ret = btrfs_commit_transaction(trans);
6031         }
6032         return ret;
6033 }
6034
6035 /*
6036  * This is somewhat expensive, updating the tree every time the
6037  * inode changes.  But, it is most likely to find the inode in cache.
6038  * FIXME, needs more benchmarking...there are no reasons other than performance
6039  * to keep or drop this code.
6040  */
6041 static int btrfs_dirty_inode(struct inode *inode)
6042 {
6043         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6044         struct btrfs_root *root = BTRFS_I(inode)->root;
6045         struct btrfs_trans_handle *trans;
6046         int ret;
6047
6048         if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
6049                 return 0;
6050
6051         trans = btrfs_join_transaction(root);
6052         if (IS_ERR(trans))
6053                 return PTR_ERR(trans);
6054
6055         ret = btrfs_update_inode(trans, root, inode);
6056         if (ret && ret == -ENOSPC) {
6057                 /* whoops, lets try again with the full transaction */
6058                 btrfs_end_transaction(trans);
6059                 trans = btrfs_start_transaction(root, 1);
6060                 if (IS_ERR(trans))
6061                         return PTR_ERR(trans);
6062
6063                 ret = btrfs_update_inode(trans, root, inode);
6064         }
6065         btrfs_end_transaction(trans);
6066         if (BTRFS_I(inode)->delayed_node)
6067                 btrfs_balance_delayed_items(fs_info);
6068
6069         return ret;
6070 }
6071
6072 /*
6073  * This is a copy of file_update_time.  We need this so we can return error on
6074  * ENOSPC for updating the inode in the case of file write and mmap writes.
6075  */
6076 static int btrfs_update_time(struct inode *inode, struct timespec *now,
6077                              int flags)
6078 {
6079         struct btrfs_root *root = BTRFS_I(inode)->root;
6080
6081         if (btrfs_root_readonly(root))
6082                 return -EROFS;
6083
6084         if (flags & S_VERSION)
6085                 inode_inc_iversion(inode);
6086         if (flags & S_CTIME)
6087                 inode->i_ctime = *now;
6088         if (flags & S_MTIME)
6089                 inode->i_mtime = *now;
6090         if (flags & S_ATIME)
6091                 inode->i_atime = *now;
6092         return btrfs_dirty_inode(inode);
6093 }
6094
6095 /*
6096  * find the highest existing sequence number in a directory
6097  * and then set the in-memory index_cnt variable to reflect
6098  * free sequence numbers
6099  */
6100 static int btrfs_set_inode_index_count(struct btrfs_inode *inode)
6101 {
6102         struct btrfs_root *root = inode->root;
6103         struct btrfs_key key, found_key;
6104         struct btrfs_path *path;
6105         struct extent_buffer *leaf;
6106         int ret;
6107
6108         key.objectid = btrfs_ino(inode);
6109         key.type = BTRFS_DIR_INDEX_KEY;
6110         key.offset = (u64)-1;
6111
6112         path = btrfs_alloc_path();
6113         if (!path)
6114                 return -ENOMEM;
6115
6116         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6117         if (ret < 0)
6118                 goto out;
6119         /* FIXME: we should be able to handle this */
6120         if (ret == 0)
6121                 goto out;
6122         ret = 0;
6123
6124         /*
6125          * MAGIC NUMBER EXPLANATION:
6126          * since we search a directory based on f_pos we have to start at 2
6127          * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
6128          * else has to start at 2
6129          */
6130         if (path->slots[0] == 0) {
6131                 inode->index_cnt = 2;
6132                 goto out;
6133         }
6134
6135         path->slots[0]--;
6136
6137         leaf = path->nodes[0];
6138         btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6139
6140         if (found_key.objectid != btrfs_ino(inode) ||
6141             found_key.type != BTRFS_DIR_INDEX_KEY) {
6142                 inode->index_cnt = 2;
6143                 goto out;
6144         }
6145
6146         inode->index_cnt = found_key.offset + 1;
6147 out:
6148         btrfs_free_path(path);
6149         return ret;
6150 }
6151
6152 /*
6153  * helper to find a free sequence number in a given directory.  This current
6154  * code is very simple, later versions will do smarter things in the btree
6155  */
6156 int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index)
6157 {
6158         int ret = 0;
6159
6160         if (dir->index_cnt == (u64)-1) {
6161                 ret = btrfs_inode_delayed_dir_index_count(dir);
6162                 if (ret) {
6163                         ret = btrfs_set_inode_index_count(dir);
6164                         if (ret)
6165                                 return ret;
6166                 }
6167         }
6168
6169         *index = dir->index_cnt;
6170         dir->index_cnt++;
6171
6172         return ret;
6173 }
6174
6175 static int btrfs_insert_inode_locked(struct inode *inode)
6176 {
6177         struct btrfs_iget_args args;
6178         args.location = &BTRFS_I(inode)->location;
6179         args.root = BTRFS_I(inode)->root;
6180
6181         return insert_inode_locked4(inode,
6182                    btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),
6183                    btrfs_find_actor, &args);
6184 }
6185
6186 /*
6187  * Inherit flags from the parent inode.
6188  *
6189  * Currently only the compression flags and the cow flags are inherited.
6190  */
6191 static void btrfs_inherit_iflags(struct inode *inode, struct inode *dir)
6192 {
6193         unsigned int flags;
6194
6195         if (!dir)
6196                 return;
6197
6198         flags = BTRFS_I(dir)->flags;
6199
6200         if (flags & BTRFS_INODE_NOCOMPRESS) {
6201                 BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS;
6202                 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
6203         } else if (flags & BTRFS_INODE_COMPRESS) {
6204                 BTRFS_I(inode)->flags &= ~BTRFS_INODE_NOCOMPRESS;
6205                 BTRFS_I(inode)->flags |= BTRFS_INODE_COMPRESS;
6206         }
6207
6208         if (flags & BTRFS_INODE_NODATACOW) {
6209                 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW;
6210                 if (S_ISREG(inode->i_mode))
6211                         BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6212         }
6213
6214         btrfs_update_iflags(inode);
6215 }
6216
6217 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans,
6218                                      struct btrfs_root *root,
6219                                      struct inode *dir,
6220                                      const char *name, int name_len,
6221                                      u64 ref_objectid, u64 objectid,
6222                                      umode_t mode, u64 *index)
6223 {
6224         struct btrfs_fs_info *fs_info = root->fs_info;
6225         struct inode *inode;
6226         struct btrfs_inode_item *inode_item;
6227         struct btrfs_key *location;
6228         struct btrfs_path *path;
6229         struct btrfs_inode_ref *ref;
6230         struct btrfs_key key[2];
6231         u32 sizes[2];
6232         int nitems = name ? 2 : 1;
6233         unsigned long ptr;
6234         int ret;
6235
6236         path = btrfs_alloc_path();
6237         if (!path)
6238                 return ERR_PTR(-ENOMEM);
6239
6240         inode = new_inode(fs_info->sb);
6241         if (!inode) {
6242                 btrfs_free_path(path);
6243                 return ERR_PTR(-ENOMEM);
6244         }
6245
6246         /*
6247          * O_TMPFILE, set link count to 0, so that after this point,
6248          * we fill in an inode item with the correct link count.
6249          */
6250         if (!name)
6251                 set_nlink(inode, 0);
6252
6253         /*
6254          * we have to initialize this early, so we can reclaim the inode
6255          * number if we fail afterwards in this function.
6256          */
6257         inode->i_ino = objectid;
6258
6259         if (dir && name) {
6260                 trace_btrfs_inode_request(dir);
6261
6262                 ret = btrfs_set_inode_index(BTRFS_I(dir), index);
6263                 if (ret) {
6264                         btrfs_free_path(path);
6265                         iput(inode);
6266                         return ERR_PTR(ret);
6267                 }
6268         } else if (dir) {
6269                 *index = 0;
6270         }
6271         /*
6272          * index_cnt is ignored for everything but a dir,
6273          * btrfs_get_inode_index_count has an explanation for the magic
6274          * number
6275          */
6276         BTRFS_I(inode)->index_cnt = 2;
6277         BTRFS_I(inode)->dir_index = *index;
6278         BTRFS_I(inode)->root = root;
6279         BTRFS_I(inode)->generation = trans->transid;
6280         inode->i_generation = BTRFS_I(inode)->generation;
6281
6282         /*
6283          * We could have gotten an inode number from somebody who was fsynced
6284          * and then removed in this same transaction, so let's just set full
6285          * sync since it will be a full sync anyway and this will blow away the
6286          * old info in the log.
6287          */
6288         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
6289
6290         key[0].objectid = objectid;
6291         key[0].type = BTRFS_INODE_ITEM_KEY;
6292         key[0].offset = 0;
6293
6294         sizes[0] = sizeof(struct btrfs_inode_item);
6295
6296         if (name) {
6297                 /*
6298                  * Start new inodes with an inode_ref. This is slightly more
6299                  * efficient for small numbers of hard links since they will
6300                  * be packed into one item. Extended refs will kick in if we
6301                  * add more hard links than can fit in the ref item.
6302                  */
6303                 key[1].objectid = objectid;
6304                 key[1].type = BTRFS_INODE_REF_KEY;
6305                 key[1].offset = ref_objectid;
6306
6307                 sizes[1] = name_len + sizeof(*ref);
6308         }
6309
6310         location = &BTRFS_I(inode)->location;
6311         location->objectid = objectid;
6312         location->offset = 0;
6313         location->type = BTRFS_INODE_ITEM_KEY;
6314
6315         ret = btrfs_insert_inode_locked(inode);
6316         if (ret < 0)
6317                 goto fail;
6318
6319         path->leave_spinning = 1;
6320         ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems);
6321         if (ret != 0)
6322                 goto fail_unlock;
6323
6324         inode_init_owner(inode, dir, mode);
6325         inode_set_bytes(inode, 0);
6326
6327         inode->i_mtime = current_time(inode);
6328         inode->i_atime = inode->i_mtime;
6329         inode->i_ctime = inode->i_mtime;
6330         BTRFS_I(inode)->i_otime = inode->i_mtime;
6331
6332         inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
6333                                   struct btrfs_inode_item);
6334         memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item,
6335                              sizeof(*inode_item));
6336         fill_inode_item(trans, path->nodes[0], inode_item, inode);
6337
6338         if (name) {
6339                 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
6340                                      struct btrfs_inode_ref);
6341                 btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len);
6342                 btrfs_set_inode_ref_index(path->nodes[0], ref, *index);
6343                 ptr = (unsigned long)(ref + 1);
6344                 write_extent_buffer(path->nodes[0], name, ptr, name_len);
6345         }
6346
6347         btrfs_mark_buffer_dirty(path->nodes[0]);
6348         btrfs_free_path(path);
6349
6350         btrfs_inherit_iflags(inode, dir);
6351
6352         if (S_ISREG(mode)) {
6353                 if (btrfs_test_opt(fs_info, NODATASUM))
6354                         BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6355                 if (btrfs_test_opt(fs_info, NODATACOW))
6356                         BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |
6357                                 BTRFS_INODE_NODATASUM;
6358         }
6359
6360         inode_tree_add(inode);
6361
6362         trace_btrfs_inode_new(inode);
6363         btrfs_set_inode_last_trans(trans, inode);
6364
6365         btrfs_update_root_times(trans, root);
6366
6367         ret = btrfs_inode_inherit_props(trans, inode, dir);
6368         if (ret)
6369                 btrfs_err(fs_info,
6370                           "error inheriting props for ino %llu (root %llu): %d",
6371                         btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, ret);
6372
6373         return inode;
6374
6375 fail_unlock:
6376         unlock_new_inode(inode);
6377 fail:
6378         if (dir && name)
6379                 BTRFS_I(dir)->index_cnt--;
6380         btrfs_free_path(path);
6381         iput(inode);
6382         return ERR_PTR(ret);
6383 }
6384
6385 static inline u8 btrfs_inode_type(struct inode *inode)
6386 {
6387         return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT];
6388 }
6389
6390 /*
6391  * utility function to add 'inode' into 'parent_inode' with
6392  * a give name and a given sequence number.
6393  * if 'add_backref' is true, also insert a backref from the
6394  * inode to the parent directory.
6395  */
6396 int btrfs_add_link(struct btrfs_trans_handle *trans,
6397                    struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
6398                    const char *name, int name_len, int add_backref, u64 index)
6399 {
6400         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
6401         int ret = 0;
6402         struct btrfs_key key;
6403         struct btrfs_root *root = parent_inode->root;
6404         u64 ino = btrfs_ino(inode);
6405         u64 parent_ino = btrfs_ino(parent_inode);
6406
6407         if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6408                 memcpy(&key, &inode->root->root_key, sizeof(key));
6409         } else {
6410                 key.objectid = ino;
6411                 key.type = BTRFS_INODE_ITEM_KEY;
6412                 key.offset = 0;
6413         }
6414
6415         if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6416                 ret = btrfs_add_root_ref(trans, fs_info, key.objectid,
6417                                          root->root_key.objectid, parent_ino,
6418                                          index, name, name_len);
6419         } else if (add_backref) {
6420                 ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino,
6421                                              parent_ino, index);
6422         }
6423
6424         /* Nothing to clean up yet */
6425         if (ret)
6426                 return ret;
6427
6428         ret = btrfs_insert_dir_item(trans, root, name, name_len,
6429                                     parent_inode, &key,
6430                                     btrfs_inode_type(&inode->vfs_inode), index);
6431         if (ret == -EEXIST || ret == -EOVERFLOW)
6432                 goto fail_dir_item;
6433         else if (ret) {
6434                 btrfs_abort_transaction(trans, ret);
6435                 return ret;
6436         }
6437
6438         btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size +
6439                            name_len * 2);
6440         inode_inc_iversion(&parent_inode->vfs_inode);
6441         parent_inode->vfs_inode.i_mtime = parent_inode->vfs_inode.i_ctime =
6442                 current_time(&parent_inode->vfs_inode);
6443         ret = btrfs_update_inode(trans, root, &parent_inode->vfs_inode);
6444         if (ret)
6445                 btrfs_abort_transaction(trans, ret);
6446         return ret;
6447
6448 fail_dir_item:
6449         if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6450                 u64 local_index;
6451                 int err;
6452                 err = btrfs_del_root_ref(trans, fs_info, key.objectid,
6453                                          root->root_key.objectid, parent_ino,
6454                                          &local_index, name, name_len);
6455
6456         } else if (add_backref) {
6457                 u64 local_index;
6458                 int err;
6459
6460                 err = btrfs_del_inode_ref(trans, root, name, name_len,
6461                                           ino, parent_ino, &local_index);
6462         }
6463         return ret;
6464 }
6465
6466 static int btrfs_add_nondir(struct btrfs_trans_handle *trans,
6467                             struct btrfs_inode *dir, struct dentry *dentry,
6468                             struct btrfs_inode *inode, int backref, u64 index)
6469 {
6470         int err = btrfs_add_link(trans, dir, inode,
6471                                  dentry->d_name.name, dentry->d_name.len,
6472                                  backref, index);
6473         if (err > 0)
6474                 err = -EEXIST;
6475         return err;
6476 }
6477
6478 static int btrfs_mknod(struct inode *dir, struct dentry *dentry,
6479                         umode_t mode, dev_t rdev)
6480 {
6481         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6482         struct btrfs_trans_handle *trans;
6483         struct btrfs_root *root = BTRFS_I(dir)->root;
6484         struct inode *inode = NULL;
6485         int err;
6486         int drop_inode = 0;
6487         u64 objectid;
6488         u64 index = 0;
6489
6490         /*
6491          * 2 for inode item and ref
6492          * 2 for dir items
6493          * 1 for xattr if selinux is on
6494          */
6495         trans = btrfs_start_transaction(root, 5);
6496         if (IS_ERR(trans))
6497                 return PTR_ERR(trans);
6498
6499         err = btrfs_find_free_ino(root, &objectid);
6500         if (err)
6501                 goto out_unlock;
6502
6503         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6504                         dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6505                         mode, &index);
6506         if (IS_ERR(inode)) {
6507                 err = PTR_ERR(inode);
6508                 goto out_unlock;
6509         }
6510
6511         /*
6512         * If the active LSM wants to access the inode during
6513         * d_instantiate it needs these. Smack checks to see
6514         * if the filesystem supports xattrs by looking at the
6515         * ops vector.
6516         */
6517         inode->i_op = &btrfs_special_inode_operations;
6518         init_special_inode(inode, inode->i_mode, rdev);
6519
6520         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6521         if (err)
6522                 goto out_unlock_inode;
6523
6524         err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6525                         0, index);
6526         if (err) {
6527                 goto out_unlock_inode;
6528         } else {
6529                 btrfs_update_inode(trans, root, inode);
6530                 unlock_new_inode(inode);
6531                 d_instantiate(dentry, inode);
6532         }
6533
6534 out_unlock:
6535         btrfs_end_transaction(trans);
6536         btrfs_balance_delayed_items(fs_info);
6537         btrfs_btree_balance_dirty(fs_info);
6538         if (drop_inode) {
6539                 inode_dec_link_count(inode);
6540                 iput(inode);
6541         }
6542         return err;
6543
6544 out_unlock_inode:
6545         drop_inode = 1;
6546         unlock_new_inode(inode);
6547         goto out_unlock;
6548
6549 }
6550
6551 static int btrfs_create(struct inode *dir, struct dentry *dentry,
6552                         umode_t mode, bool excl)
6553 {
6554         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6555         struct btrfs_trans_handle *trans;
6556         struct btrfs_root *root = BTRFS_I(dir)->root;
6557         struct inode *inode = NULL;
6558         int drop_inode_on_err = 0;
6559         int err;
6560         u64 objectid;
6561         u64 index = 0;
6562
6563         /*
6564          * 2 for inode item and ref
6565          * 2 for dir items
6566          * 1 for xattr if selinux is on
6567          */
6568         trans = btrfs_start_transaction(root, 5);
6569         if (IS_ERR(trans))
6570                 return PTR_ERR(trans);
6571
6572         err = btrfs_find_free_ino(root, &objectid);
6573         if (err)
6574                 goto out_unlock;
6575
6576         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6577                         dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6578                         mode, &index);
6579         if (IS_ERR(inode)) {
6580                 err = PTR_ERR(inode);
6581                 goto out_unlock;
6582         }
6583         drop_inode_on_err = 1;
6584         /*
6585         * If the active LSM wants to access the inode during
6586         * d_instantiate it needs these. Smack checks to see
6587         * if the filesystem supports xattrs by looking at the
6588         * ops vector.
6589         */
6590         inode->i_fop = &btrfs_file_operations;
6591         inode->i_op = &btrfs_file_inode_operations;
6592         inode->i_mapping->a_ops = &btrfs_aops;
6593
6594         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6595         if (err)
6596                 goto out_unlock_inode;
6597
6598         err = btrfs_update_inode(trans, root, inode);
6599         if (err)
6600                 goto out_unlock_inode;
6601
6602         err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6603                         0, index);
6604         if (err)
6605                 goto out_unlock_inode;
6606
6607         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
6608         unlock_new_inode(inode);
6609         d_instantiate(dentry, inode);
6610
6611 out_unlock:
6612         btrfs_end_transaction(trans);
6613         if (err && drop_inode_on_err) {
6614                 inode_dec_link_count(inode);
6615                 iput(inode);
6616         }
6617         btrfs_balance_delayed_items(fs_info);
6618         btrfs_btree_balance_dirty(fs_info);
6619         return err;
6620
6621 out_unlock_inode:
6622         unlock_new_inode(inode);
6623         goto out_unlock;
6624
6625 }
6626
6627 static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
6628                       struct dentry *dentry)
6629 {
6630         struct btrfs_trans_handle *trans = NULL;
6631         struct btrfs_root *root = BTRFS_I(dir)->root;
6632         struct inode *inode = d_inode(old_dentry);
6633         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6634         u64 index;
6635         int err;
6636         int drop_inode = 0;
6637
6638         /* do not allow sys_link's with other subvols of the same device */
6639         if (root->objectid != BTRFS_I(inode)->root->objectid)
6640                 return -EXDEV;
6641
6642         if (inode->i_nlink >= BTRFS_LINK_MAX)
6643                 return -EMLINK;
6644
6645         err = btrfs_set_inode_index(BTRFS_I(dir), &index);
6646         if (err)
6647                 goto fail;
6648
6649         /*
6650          * 2 items for inode and inode ref
6651          * 2 items for dir items
6652          * 1 item for parent inode
6653          */
6654         trans = btrfs_start_transaction(root, 5);
6655         if (IS_ERR(trans)) {
6656                 err = PTR_ERR(trans);
6657                 trans = NULL;
6658                 goto fail;
6659         }
6660
6661         /* There are several dir indexes for this inode, clear the cache. */
6662         BTRFS_I(inode)->dir_index = 0ULL;
6663         inc_nlink(inode);
6664         inode_inc_iversion(inode);
6665         inode->i_ctime = current_time(inode);
6666         ihold(inode);
6667         set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);
6668
6669         err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6670                         1, index);
6671
6672         if (err) {
6673                 drop_inode = 1;
6674         } else {
6675                 struct dentry *parent = dentry->d_parent;
6676                 err = btrfs_update_inode(trans, root, inode);
6677                 if (err)
6678                         goto fail;
6679                 if (inode->i_nlink == 1) {
6680                         /*
6681                          * If new hard link count is 1, it's a file created
6682                          * with open(2) O_TMPFILE flag.
6683                          */
6684                         err = btrfs_orphan_del(trans, BTRFS_I(inode));
6685                         if (err)
6686                                 goto fail;
6687                 }
6688                 d_instantiate(dentry, inode);
6689                 btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent);
6690         }
6691
6692         btrfs_balance_delayed_items(fs_info);
6693 fail:
6694         if (trans)
6695                 btrfs_end_transaction(trans);
6696         if (drop_inode) {
6697                 inode_dec_link_count(inode);
6698                 iput(inode);
6699         }
6700         btrfs_btree_balance_dirty(fs_info);
6701         return err;
6702 }
6703
6704 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
6705 {
6706         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6707         struct inode *inode = NULL;
6708         struct btrfs_trans_handle *trans;
6709         struct btrfs_root *root = BTRFS_I(dir)->root;
6710         int err = 0;
6711         int drop_on_err = 0;
6712         u64 objectid = 0;
6713         u64 index = 0;
6714
6715         /*
6716          * 2 items for inode and ref
6717          * 2 items for dir items
6718          * 1 for xattr if selinux is on
6719          */
6720         trans = btrfs_start_transaction(root, 5);
6721         if (IS_ERR(trans))
6722                 return PTR_ERR(trans);
6723
6724         err = btrfs_find_free_ino(root, &objectid);
6725         if (err)
6726                 goto out_fail;
6727
6728         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6729                         dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6730                         S_IFDIR | mode, &index);
6731         if (IS_ERR(inode)) {
6732                 err = PTR_ERR(inode);
6733                 goto out_fail;
6734         }
6735
6736         drop_on_err = 1;
6737         /* these must be set before we unlock the inode */
6738         inode->i_op = &btrfs_dir_inode_operations;
6739         inode->i_fop = &btrfs_dir_file_operations;
6740
6741         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6742         if (err)
6743                 goto out_fail_inode;
6744
6745         btrfs_i_size_write(BTRFS_I(inode), 0);
6746         err = btrfs_update_inode(trans, root, inode);
6747         if (err)
6748                 goto out_fail_inode;
6749
6750         err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
6751                         dentry->d_name.name,
6752                         dentry->d_name.len, 0, index);
6753         if (err)
6754                 goto out_fail_inode;
6755
6756         d_instantiate(dentry, inode);
6757         /*
6758          * mkdir is special.  We're unlocking after we call d_instantiate
6759          * to avoid a race with nfsd calling d_instantiate.
6760          */
6761         unlock_new_inode(inode);
6762         drop_on_err = 0;
6763
6764 out_fail:
6765         btrfs_end_transaction(trans);
6766         if (drop_on_err) {
6767                 inode_dec_link_count(inode);
6768                 iput(inode);
6769         }
6770         btrfs_balance_delayed_items(fs_info);
6771         btrfs_btree_balance_dirty(fs_info);
6772         return err;
6773
6774 out_fail_inode:
6775         unlock_new_inode(inode);
6776         goto out_fail;
6777 }
6778
6779 /* Find next extent map of a given extent map, caller needs to ensure locks */
6780 static struct extent_map *next_extent_map(struct extent_map *em)
6781 {
6782         struct rb_node *next;
6783
6784         next = rb_next(&em->rb_node);
6785         if (!next)
6786                 return NULL;
6787         return container_of(next, struct extent_map, rb_node);
6788 }
6789
6790 static struct extent_map *prev_extent_map(struct extent_map *em)
6791 {
6792         struct rb_node *prev;
6793
6794         prev = rb_prev(&em->rb_node);
6795         if (!prev)
6796                 return NULL;
6797         return container_of(prev, struct extent_map, rb_node);
6798 }
6799
6800 /* helper for btfs_get_extent.  Given an existing extent in the tree,
6801  * the existing extent is the nearest extent to map_start,
6802  * and an extent that you want to insert, deal with overlap and insert
6803  * the best fitted new extent into the tree.
6804  */
6805 static int merge_extent_mapping(struct extent_map_tree *em_tree,
6806                                 struct extent_map *existing,
6807                                 struct extent_map *em,
6808                                 u64 map_start)
6809 {
6810         struct extent_map *prev;
6811         struct extent_map *next;
6812         u64 start;
6813         u64 end;
6814         u64 start_diff;
6815
6816         BUG_ON(map_start < em->start || map_start >= extent_map_end(em));
6817
6818         if (existing->start > map_start) {
6819                 next = existing;
6820                 prev = prev_extent_map(next);
6821         } else {
6822                 prev = existing;
6823                 next = next_extent_map(prev);
6824         }
6825
6826         start = prev ? extent_map_end(prev) : em->start;
6827         start = max_t(u64, start, em->start);
6828         end = next ? next->start : extent_map_end(em);
6829         end = min_t(u64, end, extent_map_end(em));
6830         start_diff = start - em->start;
6831         em->start = start;
6832         em->len = end - start;
6833         if (em->block_start < EXTENT_MAP_LAST_BYTE &&
6834             !test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
6835                 em->block_start += start_diff;
6836                 em->block_len -= start_diff;
6837         }
6838         return add_extent_mapping(em_tree, em, 0);
6839 }
6840
6841 static noinline int uncompress_inline(struct btrfs_path *path,
6842                                       struct page *page,
6843                                       size_t pg_offset, u64 extent_offset,
6844                                       struct btrfs_file_extent_item *item)
6845 {
6846         int ret;
6847         struct extent_buffer *leaf = path->nodes[0];
6848         char *tmp;
6849         size_t max_size;
6850         unsigned long inline_size;
6851         unsigned long ptr;
6852         int compress_type;
6853
6854         WARN_ON(pg_offset != 0);
6855         compress_type = btrfs_file_extent_compression(leaf, item);
6856         max_size = btrfs_file_extent_ram_bytes(leaf, item);
6857         inline_size = btrfs_file_extent_inline_item_len(leaf,
6858                                         btrfs_item_nr(path->slots[0]));
6859         tmp = kmalloc(inline_size, GFP_NOFS);
6860         if (!tmp)
6861                 return -ENOMEM;
6862         ptr = btrfs_file_extent_inline_start(item);
6863
6864         read_extent_buffer(leaf, tmp, ptr, inline_size);
6865
6866         max_size = min_t(unsigned long, PAGE_SIZE, max_size);
6867         ret = btrfs_decompress(compress_type, tmp, page,
6868                                extent_offset, inline_size, max_size);
6869
6870         /*
6871          * decompression code contains a memset to fill in any space between the end
6872          * of the uncompressed data and the end of max_size in case the decompressed
6873          * data ends up shorter than ram_bytes.  That doesn't cover the hole between
6874          * the end of an inline extent and the beginning of the next block, so we
6875          * cover that region here.
6876          */
6877
6878         if (max_size + pg_offset < PAGE_SIZE) {
6879                 char *map = kmap(page);
6880                 memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset);
6881                 kunmap(page);
6882         }
6883         kfree(tmp);
6884         return ret;
6885 }
6886
6887 /*
6888  * a bit scary, this does extent mapping from logical file offset to the disk.
6889  * the ugly parts come from merging extents from the disk with the in-ram
6890  * representation.  This gets more complex because of the data=ordered code,
6891  * where the in-ram extents might be locked pending data=ordered completion.
6892  *
6893  * This also copies inline extents directly into the page.
6894  */
6895 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
6896                 struct page *page,
6897             size_t pg_offset, u64 start, u64 len,
6898                 int create)
6899 {
6900         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
6901         int ret;
6902         int err = 0;
6903         u64 extent_start = 0;
6904         u64 extent_end = 0;
6905         u64 objectid = btrfs_ino(inode);
6906         u32 found_type;
6907         struct btrfs_path *path = NULL;
6908         struct btrfs_root *root = inode->root;
6909         struct btrfs_file_extent_item *item;
6910         struct extent_buffer *leaf;
6911         struct btrfs_key found_key;
6912         struct extent_map *em = NULL;
6913         struct extent_map_tree *em_tree = &inode->extent_tree;
6914         struct extent_io_tree *io_tree = &inode->io_tree;
6915         struct btrfs_trans_handle *trans = NULL;
6916         const bool new_inline = !page || create;
6917
6918 again:
6919         read_lock(&em_tree->lock);
6920         em = lookup_extent_mapping(em_tree, start, len);
6921         if (em)
6922                 em->bdev = fs_info->fs_devices->latest_bdev;
6923         read_unlock(&em_tree->lock);
6924
6925         if (em) {
6926                 if (em->start > start || em->start + em->len <= start)
6927                         free_extent_map(em);
6928                 else if (em->block_start == EXTENT_MAP_INLINE && page)
6929                         free_extent_map(em);
6930                 else
6931                         goto out;
6932         }
6933         em = alloc_extent_map();
6934         if (!em) {
6935                 err = -ENOMEM;
6936                 goto out;
6937         }
6938         em->bdev = fs_info->fs_devices->latest_bdev;
6939         em->start = EXTENT_MAP_HOLE;
6940         em->orig_start = EXTENT_MAP_HOLE;
6941         em->len = (u64)-1;
6942         em->block_len = (u64)-1;
6943
6944         if (!path) {
6945                 path = btrfs_alloc_path();
6946                 if (!path) {
6947                         err = -ENOMEM;
6948                         goto out;
6949                 }
6950                 /*
6951                  * Chances are we'll be called again, so go ahead and do
6952                  * readahead
6953                  */
6954                 path->reada = READA_FORWARD;
6955         }
6956
6957         ret = btrfs_lookup_file_extent(trans, root, path,
6958                                        objectid, start, trans != NULL);
6959         if (ret < 0) {
6960                 err = ret;
6961                 goto out;
6962         }
6963
6964         if (ret != 0) {
6965                 if (path->slots[0] == 0)
6966                         goto not_found;
6967                 path->slots[0]--;
6968         }
6969
6970         leaf = path->nodes[0];
6971         item = btrfs_item_ptr(leaf, path->slots[0],
6972                               struct btrfs_file_extent_item);
6973         /* are we inside the extent that was found? */
6974         btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6975         found_type = found_key.type;
6976         if (found_key.objectid != objectid ||
6977             found_type != BTRFS_EXTENT_DATA_KEY) {
6978                 /*
6979                  * If we backup past the first extent we want to move forward
6980                  * and see if there is an extent in front of us, otherwise we'll
6981                  * say there is a hole for our whole search range which can
6982                  * cause problems.
6983                  */
6984                 extent_end = start;
6985                 goto next;
6986         }
6987
6988         found_type = btrfs_file_extent_type(leaf, item);
6989         extent_start = found_key.offset;
6990         if (found_type == BTRFS_FILE_EXTENT_REG ||
6991             found_type == BTRFS_FILE_EXTENT_PREALLOC) {
6992                 extent_end = extent_start +
6993                        btrfs_file_extent_num_bytes(leaf, item);
6994
6995                 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item,
6996                                                        extent_start);
6997         } else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
6998                 size_t size;
6999                 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
7000                 extent_end = ALIGN(extent_start + size,
7001                                    fs_info->sectorsize);
7002
7003                 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item,
7004                                                       path->slots[0],
7005                                                       extent_start);
7006         }
7007 next:
7008         if (start >= extent_end) {
7009                 path->slots[0]++;
7010                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
7011                         ret = btrfs_next_leaf(root, path);
7012                         if (ret < 0) {
7013                                 err = ret;
7014                                 goto out;
7015                         }
7016                         if (ret > 0)
7017                                 goto not_found;
7018                         leaf = path->nodes[0];
7019                 }
7020                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7021                 if (found_key.objectid != objectid ||
7022                     found_key.type != BTRFS_EXTENT_DATA_KEY)
7023                         goto not_found;
7024                 if (start + len <= found_key.offset)
7025                         goto not_found;
7026                 if (start > found_key.offset)
7027                         goto next;
7028                 em->start = start;
7029                 em->orig_start = start;
7030                 em->len = found_key.offset - start;
7031                 goto not_found_em;
7032         }
7033
7034         btrfs_extent_item_to_extent_map(inode, path, item,
7035                         new_inline, em);
7036
7037         if (found_type == BTRFS_FILE_EXTENT_REG ||
7038             found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7039                 goto insert;
7040         } else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
7041                 unsigned long ptr;
7042                 char *map;
7043                 size_t size;
7044                 size_t extent_offset;
7045                 size_t copy_size;
7046
7047                 if (new_inline)
7048                         goto out;
7049
7050                 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
7051                 extent_offset = page_offset(page) + pg_offset - extent_start;
7052                 copy_size = min_t(u64, PAGE_SIZE - pg_offset,
7053                                   size - extent_offset);
7054                 em->start = extent_start + extent_offset;
7055                 em->len = ALIGN(copy_size, fs_info->sectorsize);
7056                 em->orig_block_len = em->len;
7057                 em->orig_start = em->start;
7058                 ptr = btrfs_file_extent_inline_start(item) + extent_offset;
7059                 if (create == 0 && !PageUptodate(page)) {
7060                         if (btrfs_file_extent_compression(leaf, item) !=
7061                             BTRFS_COMPRESS_NONE) {
7062                                 ret = uncompress_inline(path, page, pg_offset,
7063                                                         extent_offset, item);
7064                                 if (ret) {
7065                                         err = ret;
7066                                         goto out;
7067                                 }
7068                         } else {
7069                                 map = kmap(page);
7070                                 read_extent_buffer(leaf, map + pg_offset, ptr,
7071                                                    copy_size);
7072                                 if (pg_offset + copy_size < PAGE_SIZE) {
7073                                         memset(map + pg_offset + copy_size, 0,
7074                                                PAGE_SIZE - pg_offset -
7075                                                copy_size);
7076                                 }
7077                                 kunmap(page);
7078                         }
7079                         flush_dcache_page(page);
7080                 } else if (create && PageUptodate(page)) {
7081                         BUG();
7082                         if (!trans) {
7083                                 kunmap(page);
7084                                 free_extent_map(em);
7085                                 em = NULL;
7086
7087                                 btrfs_release_path(path);
7088                                 trans = btrfs_join_transaction(root);
7089
7090                                 if (IS_ERR(trans))
7091                                         return ERR_CAST(trans);
7092                                 goto again;
7093                         }
7094                         map = kmap(page);
7095                         write_extent_buffer(leaf, map + pg_offset, ptr,
7096                                             copy_size);
7097                         kunmap(page);
7098                         btrfs_mark_buffer_dirty(leaf);
7099                 }
7100                 set_extent_uptodate(io_tree, em->start,
7101                                     extent_map_end(em) - 1, NULL, GFP_NOFS);
7102                 goto insert;
7103         }
7104 not_found:
7105         em->start = start;
7106         em->orig_start = start;
7107         em->len = len;
7108 not_found_em:
7109         em->block_start = EXTENT_MAP_HOLE;
7110         set_bit(EXTENT_FLAG_VACANCY, &em->flags);
7111 insert:
7112         btrfs_release_path(path);
7113         if (em->start > start || extent_map_end(em) <= start) {
7114                 btrfs_err(fs_info,
7115                           "bad extent! em: [%llu %llu] passed [%llu %llu]",
7116                           em->start, em->len, start, len);
7117                 err = -EIO;
7118                 goto out;
7119         }
7120
7121         err = 0;
7122         write_lock(&em_tree->lock);
7123         ret = add_extent_mapping(em_tree, em, 0);
7124         /* it is possible that someone inserted the extent into the tree
7125          * while we had the lock dropped.  It is also possible that
7126          * an overlapping map exists in the tree
7127          */
7128         if (ret == -EEXIST) {
7129                 struct extent_map *existing;
7130
7131                 ret = 0;
7132
7133                 existing = search_extent_mapping(em_tree, start, len);
7134                 /*
7135                  * existing will always be non-NULL, since there must be
7136                  * extent causing the -EEXIST.
7137                  */
7138                 if (existing->start == em->start &&
7139                     extent_map_end(existing) >= extent_map_end(em) &&
7140                     em->block_start == existing->block_start) {
7141                         /*
7142                          * The existing extent map already encompasses the
7143                          * entire extent map we tried to add.
7144                          */
7145                         free_extent_map(em);
7146                         em = existing;
7147                         err = 0;
7148
7149                 } else if (start >= extent_map_end(existing) ||
7150                     start <= existing->start) {
7151                         /*
7152                          * The existing extent map is the one nearest to
7153                          * the [start, start + len) range which overlaps
7154                          */
7155                         err = merge_extent_mapping(em_tree, existing,
7156                                                    em, start);
7157                         free_extent_map(existing);
7158                         if (err) {
7159                                 free_extent_map(em);
7160                                 em = NULL;
7161                         }
7162                 } else {
7163                         free_extent_map(em);
7164                         em = existing;
7165                         err = 0;
7166                 }
7167         }
7168         write_unlock(&em_tree->lock);
7169 out:
7170
7171         trace_btrfs_get_extent(root, inode, em);
7172
7173         btrfs_free_path(path);
7174         if (trans) {
7175                 ret = btrfs_end_transaction(trans);
7176                 if (!err)
7177                         err = ret;
7178         }
7179         if (err) {
7180                 free_extent_map(em);
7181                 return ERR_PTR(err);
7182         }
7183         BUG_ON(!em); /* Error is always set */
7184         return em;
7185 }
7186
7187 struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode,
7188                 struct page *page,
7189                 size_t pg_offset, u64 start, u64 len,
7190                 int create)
7191 {
7192         struct extent_map *em;
7193         struct extent_map *hole_em = NULL;
7194         u64 range_start = start;
7195         u64 end;
7196         u64 found;
7197         u64 found_end;
7198         int err = 0;
7199
7200         em = btrfs_get_extent(inode, page, pg_offset, start, len, create);
7201         if (IS_ERR(em))
7202                 return em;
7203         /*
7204          * If our em maps to:
7205          * - a hole or
7206          * - a pre-alloc extent,
7207          * there might actually be delalloc bytes behind it.
7208          */
7209         if (em->block_start != EXTENT_MAP_HOLE &&
7210             !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7211                 return em;
7212         else
7213                 hole_em = em;
7214
7215         /* check to see if we've wrapped (len == -1 or similar) */
7216         end = start + len;
7217         if (end < start)
7218                 end = (u64)-1;
7219         else
7220                 end -= 1;
7221
7222         em = NULL;
7223
7224         /* ok, we didn't find anything, lets look for delalloc */
7225         found = count_range_bits(&inode->io_tree, &range_start,
7226                                  end, len, EXTENT_DELALLOC, 1);
7227         found_end = range_start + found;
7228         if (found_end < range_start)
7229                 found_end = (u64)-1;
7230
7231         /*
7232          * we didn't find anything useful, return
7233          * the original results from get_extent()
7234          */
7235         if (range_start > end || found_end <= start) {
7236                 em = hole_em;
7237                 hole_em = NULL;
7238                 goto out;
7239         }
7240
7241         /* adjust the range_start to make sure it doesn't
7242          * go backwards from the start they passed in
7243          */
7244         range_start = max(start, range_start);
7245         found = found_end - range_start;
7246
7247         if (found > 0) {
7248                 u64 hole_start = start;
7249                 u64 hole_len = len;
7250
7251                 em = alloc_extent_map();
7252                 if (!em) {
7253                         err = -ENOMEM;
7254                         goto out;
7255                 }
7256                 /*
7257                  * when btrfs_get_extent can't find anything it
7258                  * returns one huge hole
7259                  *
7260                  * make sure what it found really fits our range, and
7261                  * adjust to make sure it is based on the start from
7262                  * the caller
7263                  */
7264                 if (hole_em) {
7265                         u64 calc_end = extent_map_end(hole_em);
7266
7267                         if (calc_end <= start || (hole_em->start > end)) {
7268                                 free_extent_map(hole_em);
7269                                 hole_em = NULL;
7270                         } else {
7271                                 hole_start = max(hole_em->start, start);
7272                                 hole_len = calc_end - hole_start;
7273                         }
7274                 }
7275                 em->bdev = NULL;
7276                 if (hole_em && range_start > hole_start) {
7277                         /* our hole starts before our delalloc, so we
7278                          * have to return just the parts of the hole
7279                          * that go until  the delalloc starts
7280                          */
7281                         em->len = min(hole_len,
7282                                       range_start - hole_start);
7283                         em->start = hole_start;
7284                         em->orig_start = hole_start;
7285                         /*
7286                          * don't adjust block start at all,
7287                          * it is fixed at EXTENT_MAP_HOLE
7288                          */
7289                         em->block_start = hole_em->block_start;
7290                         em->block_len = hole_len;
7291                         if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags))
7292                                 set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
7293                 } else {
7294                         em->start = range_start;
7295                         em->len = found;
7296                         em->orig_start = range_start;
7297                         em->block_start = EXTENT_MAP_DELALLOC;
7298                         em->block_len = found;
7299                 }
7300         } else if (hole_em) {
7301                 return hole_em;
7302         }
7303 out:
7304
7305         free_extent_map(hole_em);
7306         if (err) {
7307                 free_extent_map(em);
7308                 return ERR_PTR(err);
7309         }
7310         return em;
7311 }
7312
7313 static struct extent_map *btrfs_create_dio_extent(struct inode *inode,
7314                                                   const u64 start,
7315                                                   const u64 len,
7316                                                   const u64 orig_start,
7317                                                   const u64 block_start,
7318                                                   const u64 block_len,
7319                                                   const u64 orig_block_len,
7320                                                   const u64 ram_bytes,
7321                                                   const int type)
7322 {
7323         struct extent_map *em = NULL;
7324         int ret;
7325
7326         if (type != BTRFS_ORDERED_NOCOW) {
7327                 em = create_io_em(inode, start, len, orig_start,
7328                                   block_start, block_len, orig_block_len,
7329                                   ram_bytes,
7330                                   BTRFS_COMPRESS_NONE, /* compress_type */
7331                                   type);
7332                 if (IS_ERR(em))
7333                         goto out;
7334         }
7335         ret = btrfs_add_ordered_extent_dio(inode, start, block_start,
7336                                            len, block_len, type);
7337         if (ret) {
7338                 if (em) {
7339                         free_extent_map(em);
7340                         btrfs_drop_extent_cache(BTRFS_I(inode), start,
7341                                                 start + len - 1, 0);
7342                 }
7343                 em = ERR_PTR(ret);
7344         }
7345  out:
7346
7347         return em;
7348 }
7349
7350 static struct extent_map *btrfs_new_extent_direct(struct inode *inode,
7351                                                   u64 start, u64 len)
7352 {
7353         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7354         struct btrfs_root *root = BTRFS_I(inode)->root;
7355         struct extent_map *em;
7356         struct btrfs_key ins;
7357         u64 alloc_hint;
7358         int ret;
7359
7360         alloc_hint = get_extent_allocation_hint(inode, start, len);
7361         ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize,
7362                                    0, alloc_hint, &ins, 1, 1);
7363         if (ret)
7364                 return ERR_PTR(ret);
7365
7366         em = btrfs_create_dio_extent(inode, start, ins.offset, start,
7367                                      ins.objectid, ins.offset, ins.offset,
7368                                      ins.offset, BTRFS_ORDERED_REGULAR);
7369         btrfs_dec_block_group_reservations(fs_info, ins.objectid);
7370         if (IS_ERR(em))
7371                 btrfs_free_reserved_extent(fs_info, ins.objectid,
7372                                            ins.offset, 1);
7373
7374         return em;
7375 }
7376
7377 /*
7378  * returns 1 when the nocow is safe, < 1 on error, 0 if the
7379  * block must be cow'd
7380  */
7381 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
7382                               u64 *orig_start, u64 *orig_block_len,
7383                               u64 *ram_bytes)
7384 {
7385         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7386         struct btrfs_path *path;
7387         int ret;
7388         struct extent_buffer *leaf;
7389         struct btrfs_root *root = BTRFS_I(inode)->root;
7390         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7391         struct btrfs_file_extent_item *fi;
7392         struct btrfs_key key;
7393         u64 disk_bytenr;
7394         u64 backref_offset;
7395         u64 extent_end;
7396         u64 num_bytes;
7397         int slot;
7398         int found_type;
7399         bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW);
7400
7401         path = btrfs_alloc_path();
7402         if (!path)
7403                 return -ENOMEM;
7404
7405         ret = btrfs_lookup_file_extent(NULL, root, path,
7406                         btrfs_ino(BTRFS_I(inode)), offset, 0);
7407         if (ret < 0)
7408                 goto out;
7409
7410         slot = path->slots[0];
7411         if (ret == 1) {
7412                 if (slot == 0) {
7413                         /* can't find the item, must cow */
7414                         ret = 0;
7415                         goto out;
7416                 }
7417                 slot--;
7418         }
7419         ret = 0;
7420         leaf = path->nodes[0];
7421         btrfs_item_key_to_cpu(leaf, &key, slot);
7422         if (key.objectid != btrfs_ino(BTRFS_I(inode)) ||
7423             key.type != BTRFS_EXTENT_DATA_KEY) {
7424                 /* not our file or wrong item type, must cow */
7425                 goto out;
7426         }
7427
7428         if (key.offset > offset) {
7429                 /* Wrong offset, must cow */
7430                 goto out;
7431         }
7432
7433         fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
7434         found_type = btrfs_file_extent_type(leaf, fi);
7435         if (found_type != BTRFS_FILE_EXTENT_REG &&
7436             found_type != BTRFS_FILE_EXTENT_PREALLOC) {
7437                 /* not a regular extent, must cow */
7438                 goto out;
7439         }
7440
7441         if (!nocow && found_type == BTRFS_FILE_EXTENT_REG)
7442                 goto out;
7443
7444         extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
7445         if (extent_end <= offset)
7446                 goto out;
7447
7448         disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
7449         if (disk_bytenr == 0)
7450                 goto out;
7451
7452         if (btrfs_file_extent_compression(leaf, fi) ||
7453             btrfs_file_extent_encryption(leaf, fi) ||
7454             btrfs_file_extent_other_encoding(leaf, fi))
7455                 goto out;
7456
7457         backref_offset = btrfs_file_extent_offset(leaf, fi);
7458
7459         if (orig_start) {
7460                 *orig_start = key.offset - backref_offset;
7461                 *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
7462                 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
7463         }
7464
7465         if (btrfs_extent_readonly(fs_info, disk_bytenr))
7466                 goto out;
7467
7468         num_bytes = min(offset + *len, extent_end) - offset;
7469         if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7470                 u64 range_end;
7471
7472                 range_end = round_up(offset + num_bytes,
7473                                      root->fs_info->sectorsize) - 1;
7474                 ret = test_range_bit(io_tree, offset, range_end,
7475                                      EXTENT_DELALLOC, 0, NULL);
7476                 if (ret) {
7477                         ret = -EAGAIN;
7478                         goto out;
7479                 }
7480         }
7481
7482         btrfs_release_path(path);
7483
7484         /*
7485          * look for other files referencing this extent, if we
7486          * find any we must cow
7487          */
7488
7489         ret = btrfs_cross_ref_exist(root, btrfs_ino(BTRFS_I(inode)),
7490                                     key.offset - backref_offset, disk_bytenr);
7491         if (ret) {
7492                 ret = 0;
7493                 goto out;
7494         }
7495
7496         /*
7497          * adjust disk_bytenr and num_bytes to cover just the bytes
7498          * in this extent we are about to write.  If there
7499          * are any csums in that range we have to cow in order
7500          * to keep the csums correct
7501          */
7502         disk_bytenr += backref_offset;
7503         disk_bytenr += offset - key.offset;
7504         if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes))
7505                 goto out;
7506         /*
7507          * all of the above have passed, it is safe to overwrite this extent
7508          * without cow
7509          */
7510         *len = num_bytes;
7511         ret = 1;
7512 out:
7513         btrfs_free_path(path);
7514         return ret;
7515 }
7516
7517 bool btrfs_page_exists_in_range(struct inode *inode, loff_t start, loff_t end)
7518 {
7519         struct radix_tree_root *root = &inode->i_mapping->page_tree;
7520         bool found = false;
7521         void **pagep = NULL;
7522         struct page *page = NULL;
7523         unsigned long start_idx;
7524         unsigned long end_idx;
7525
7526         start_idx = start >> PAGE_SHIFT;
7527
7528         /*
7529          * end is the last byte in the last page.  end == start is legal
7530          */
7531         end_idx = end >> PAGE_SHIFT;
7532
7533         rcu_read_lock();
7534
7535         /* Most of the code in this while loop is lifted from
7536          * find_get_page.  It's been modified to begin searching from a
7537          * page and return just the first page found in that range.  If the
7538          * found idx is less than or equal to the end idx then we know that
7539          * a page exists.  If no pages are found or if those pages are
7540          * outside of the range then we're fine (yay!) */
7541         while (page == NULL &&
7542                radix_tree_gang_lookup_slot(root, &pagep, NULL, start_idx, 1)) {
7543                 page = radix_tree_deref_slot(pagep);
7544                 if (unlikely(!page))
7545                         break;
7546
7547                 if (radix_tree_exception(page)) {
7548                         if (radix_tree_deref_retry(page)) {
7549                                 page = NULL;
7550                                 continue;
7551                         }
7552                         /*
7553                          * Otherwise, shmem/tmpfs must be storing a swap entry
7554                          * here as an exceptional entry: so return it without
7555                          * attempting to raise page count.
7556                          */
7557                         page = NULL;
7558                         break; /* TODO: Is this relevant for this use case? */
7559                 }
7560
7561                 if (!page_cache_get_speculative(page)) {
7562                         page = NULL;
7563                         continue;
7564                 }
7565
7566                 /*
7567                  * Has the page moved?
7568                  * This is part of the lockless pagecache protocol. See
7569                  * include/linux/pagemap.h for details.
7570                  */
7571                 if (unlikely(page != *pagep)) {
7572                         put_page(page);
7573                         page = NULL;
7574                 }
7575         }
7576
7577         if (page) {
7578                 if (page->index <= end_idx)
7579                         found = true;
7580                 put_page(page);
7581         }
7582
7583         rcu_read_unlock();
7584         return found;
7585 }
7586
7587 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
7588                               struct extent_state **cached_state, int writing)
7589 {
7590         struct btrfs_ordered_extent *ordered;
7591         int ret = 0;
7592
7593         while (1) {
7594                 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7595                                  cached_state);
7596                 /*
7597                  * We're concerned with the entire range that we're going to be
7598                  * doing DIO to, so we need to make sure there's no ordered
7599                  * extents in this range.
7600                  */
7601                 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,
7602                                                      lockend - lockstart + 1);
7603
7604                 /*
7605                  * We need to make sure there are no buffered pages in this
7606                  * range either, we could have raced between the invalidate in
7607                  * generic_file_direct_write and locking the extent.  The
7608                  * invalidate needs to happen so that reads after a write do not
7609                  * get stale data.
7610                  */
7611                 if (!ordered &&
7612                     (!writing ||
7613                      !btrfs_page_exists_in_range(inode, lockstart, lockend)))
7614                         break;
7615
7616                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7617                                      cached_state, GFP_NOFS);
7618
7619                 if (ordered) {
7620                         /*
7621                          * If we are doing a DIO read and the ordered extent we
7622                          * found is for a buffered write, we can not wait for it
7623                          * to complete and retry, because if we do so we can
7624                          * deadlock with concurrent buffered writes on page
7625                          * locks. This happens only if our DIO read covers more
7626                          * than one extent map, if at this point has already
7627                          * created an ordered extent for a previous extent map
7628                          * and locked its range in the inode's io tree, and a
7629                          * concurrent write against that previous extent map's
7630                          * range and this range started (we unlock the ranges
7631                          * in the io tree only when the bios complete and
7632                          * buffered writes always lock pages before attempting
7633                          * to lock range in the io tree).
7634                          */
7635                         if (writing ||
7636                             test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))
7637                                 btrfs_start_ordered_extent(inode, ordered, 1);
7638                         else
7639                                 ret = -ENOTBLK;
7640                         btrfs_put_ordered_extent(ordered);
7641                 } else {
7642                         /*
7643                          * We could trigger writeback for this range (and wait
7644                          * for it to complete) and then invalidate the pages for
7645                          * this range (through invalidate_inode_pages2_range()),
7646                          * but that can lead us to a deadlock with a concurrent
7647                          * call to readpages() (a buffered read or a defrag call
7648                          * triggered a readahead) on a page lock due to an
7649                          * ordered dio extent we created before but did not have
7650                          * yet a corresponding bio submitted (whence it can not
7651                          * complete), which makes readpages() wait for that
7652                          * ordered extent to complete while holding a lock on
7653                          * that page.
7654                          */
7655                         ret = -ENOTBLK;
7656                 }
7657
7658                 if (ret)
7659                         break;
7660
7661                 cond_resched();
7662         }
7663
7664         return ret;
7665 }
7666
7667 /* The callers of this must take lock_extent() */
7668 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len,
7669                                        u64 orig_start, u64 block_start,
7670                                        u64 block_len, u64 orig_block_len,
7671                                        u64 ram_bytes, int compress_type,
7672                                        int type)
7673 {
7674         struct extent_map_tree *em_tree;
7675         struct extent_map *em;
7676         struct btrfs_root *root = BTRFS_I(inode)->root;
7677         int ret;
7678
7679         ASSERT(type == BTRFS_ORDERED_PREALLOC ||
7680                type == BTRFS_ORDERED_COMPRESSED ||
7681                type == BTRFS_ORDERED_NOCOW ||
7682                type == BTRFS_ORDERED_REGULAR);
7683
7684         em_tree = &BTRFS_I(inode)->extent_tree;
7685         em = alloc_extent_map();
7686         if (!em)
7687                 return ERR_PTR(-ENOMEM);
7688
7689         em->start = start;
7690         em->orig_start = orig_start;
7691         em->len = len;
7692         em->block_len = block_len;
7693         em->block_start = block_start;
7694         em->bdev = root->fs_info->fs_devices->latest_bdev;
7695         em->orig_block_len = orig_block_len;
7696         em->ram_bytes = ram_bytes;
7697         em->generation = -1;
7698         set_bit(EXTENT_FLAG_PINNED, &em->flags);
7699         if (type == BTRFS_ORDERED_PREALLOC) {
7700                 set_bit(EXTENT_FLAG_FILLING, &em->flags);
7701         } else if (type == BTRFS_ORDERED_COMPRESSED) {
7702                 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
7703                 em->compress_type = compress_type;
7704         }
7705
7706         do {
7707                 btrfs_drop_extent_cache(BTRFS_I(inode), em->start,
7708                                 em->start + em->len - 1, 0);
7709                 write_lock(&em_tree->lock);
7710                 ret = add_extent_mapping(em_tree, em, 1);
7711                 write_unlock(&em_tree->lock);
7712                 /*
7713                  * The caller has taken lock_extent(), who could race with us
7714                  * to add em?
7715                  */
7716         } while (ret == -EEXIST);
7717
7718         if (ret) {
7719                 free_extent_map(em);
7720                 return ERR_PTR(ret);
7721         }
7722
7723         /* em got 2 refs now, callers needs to do free_extent_map once. */
7724         return em;
7725 }
7726
7727 static void adjust_dio_outstanding_extents(struct inode *inode,
7728                                            struct btrfs_dio_data *dio_data,
7729                                            const u64 len)
7730 {
7731         unsigned num_extents = count_max_extents(len);
7732
7733         /*
7734          * If we have an outstanding_extents count still set then we're
7735          * within our reservation, otherwise we need to adjust our inode
7736          * counter appropriately.
7737          */
7738         if (dio_data->outstanding_extents >= num_extents) {
7739                 dio_data->outstanding_extents -= num_extents;
7740         } else {
7741                 /*
7742                  * If dio write length has been split due to no large enough
7743                  * contiguous space, we need to compensate our inode counter
7744                  * appropriately.
7745                  */
7746                 u64 num_needed = num_extents - dio_data->outstanding_extents;
7747
7748                 spin_lock(&BTRFS_I(inode)->lock);
7749                 BTRFS_I(inode)->outstanding_extents += num_needed;
7750                 spin_unlock(&BTRFS_I(inode)->lock);
7751         }
7752 }
7753
7754 static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock,
7755                                    struct buffer_head *bh_result, int create)
7756 {
7757         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7758         struct extent_map *em;
7759         struct extent_state *cached_state = NULL;
7760         struct btrfs_dio_data *dio_data = NULL;
7761         u64 start = iblock << inode->i_blkbits;
7762         u64 lockstart, lockend;
7763         u64 len = bh_result->b_size;
7764         int unlock_bits = EXTENT_LOCKED;
7765         int ret = 0;
7766
7767         if (create)
7768                 unlock_bits |= EXTENT_DIRTY;
7769         else
7770                 len = min_t(u64, len, fs_info->sectorsize);
7771
7772         lockstart = start;
7773         lockend = start + len - 1;
7774
7775         if (current->journal_info) {
7776                 /*
7777                  * Need to pull our outstanding extents and set journal_info to NULL so
7778                  * that anything that needs to check if there's a transaction doesn't get
7779                  * confused.
7780                  */
7781                 dio_data = current->journal_info;
7782                 current->journal_info = NULL;
7783         }
7784
7785         /*
7786          * If this errors out it's because we couldn't invalidate pagecache for
7787          * this range and we need to fallback to buffered.
7788          */
7789         if (lock_extent_direct(inode, lockstart, lockend, &cached_state,
7790                                create)) {
7791                 ret = -ENOTBLK;
7792                 goto err;
7793         }
7794
7795         em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len, 0);
7796         if (IS_ERR(em)) {
7797                 ret = PTR_ERR(em);
7798                 goto unlock_err;
7799         }
7800
7801         /*
7802          * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
7803          * io.  INLINE is special, and we could probably kludge it in here, but
7804          * it's still buffered so for safety lets just fall back to the generic
7805          * buffered path.
7806          *
7807          * For COMPRESSED we _have_ to read the entire extent in so we can
7808          * decompress it, so there will be buffering required no matter what we
7809          * do, so go ahead and fallback to buffered.
7810          *
7811          * We return -ENOTBLK because that's what makes DIO go ahead and go back
7812          * to buffered IO.  Don't blame me, this is the price we pay for using
7813          * the generic code.
7814          */
7815         if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) ||
7816             em->block_start == EXTENT_MAP_INLINE) {
7817                 free_extent_map(em);
7818                 ret = -ENOTBLK;
7819                 goto unlock_err;
7820         }
7821
7822         /* Just a good old fashioned hole, return */
7823         if (!create && (em->block_start == EXTENT_MAP_HOLE ||
7824                         test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
7825                 free_extent_map(em);
7826                 goto unlock_err;
7827         }
7828
7829         /*
7830          * We don't allocate a new extent in the following cases
7831          *
7832          * 1) The inode is marked as NODATACOW.  In this case we'll just use the
7833          * existing extent.
7834          * 2) The extent is marked as PREALLOC.  We're good to go here and can
7835          * just use the extent.
7836          *
7837          */
7838         if (!create) {
7839                 len = min(len, em->len - (start - em->start));
7840                 lockstart = start + len;
7841                 goto unlock;
7842         }
7843
7844         if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) ||
7845             ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
7846              em->block_start != EXTENT_MAP_HOLE)) {
7847                 int type;
7848                 u64 block_start, orig_start, orig_block_len, ram_bytes;
7849
7850                 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7851                         type = BTRFS_ORDERED_PREALLOC;
7852                 else
7853                         type = BTRFS_ORDERED_NOCOW;
7854                 len = min(len, em->len - (start - em->start));
7855                 block_start = em->block_start + (start - em->start);
7856
7857                 if (can_nocow_extent(inode, start, &len, &orig_start,
7858                                      &orig_block_len, &ram_bytes) == 1 &&
7859                     btrfs_inc_nocow_writers(fs_info, block_start)) {
7860                         struct extent_map *em2;
7861
7862                         em2 = btrfs_create_dio_extent(inode, start, len,
7863                                                       orig_start, block_start,
7864                                                       len, orig_block_len,
7865                                                       ram_bytes, type);
7866                         btrfs_dec_nocow_writers(fs_info, block_start);
7867                         if (type == BTRFS_ORDERED_PREALLOC) {
7868                                 free_extent_map(em);
7869                                 em = em2;
7870                         }
7871                         if (em2 && IS_ERR(em2)) {
7872                                 ret = PTR_ERR(em2);
7873                                 goto unlock_err;
7874                         }
7875                         /*
7876                          * For inode marked NODATACOW or extent marked PREALLOC,
7877                          * use the existing or preallocated extent, so does not
7878                          * need to adjust btrfs_space_info's bytes_may_use.
7879                          */
7880                         btrfs_free_reserved_data_space_noquota(inode,
7881                                         start, len);
7882                         goto unlock;
7883                 }
7884         }
7885
7886         /*
7887          * this will cow the extent, reset the len in case we changed
7888          * it above
7889          */
7890         len = bh_result->b_size;
7891         free_extent_map(em);
7892         em = btrfs_new_extent_direct(inode, start, len);
7893         if (IS_ERR(em)) {
7894                 ret = PTR_ERR(em);
7895                 goto unlock_err;
7896         }
7897         len = min(len, em->len - (start - em->start));
7898 unlock:
7899         bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
7900                 inode->i_blkbits;
7901         bh_result->b_size = len;
7902         bh_result->b_bdev = em->bdev;
7903         set_buffer_mapped(bh_result);
7904         if (create) {
7905                 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7906                         set_buffer_new(bh_result);
7907
7908                 /*
7909                  * Need to update the i_size under the extent lock so buffered
7910                  * readers will get the updated i_size when we unlock.
7911                  */
7912                 if (!dio_data->overwrite && start + len > i_size_read(inode))
7913                         i_size_write(inode, start + len);
7914
7915                 adjust_dio_outstanding_extents(inode, dio_data, len);
7916                 WARN_ON(dio_data->reserve < len);
7917                 dio_data->reserve -= len;
7918                 dio_data->unsubmitted_oe_range_end = start + len;
7919                 current->journal_info = dio_data;
7920         }
7921
7922         /*
7923          * In the case of write we need to clear and unlock the entire range,
7924          * in the case of read we need to unlock only the end area that we
7925          * aren't using if there is any left over space.
7926          */
7927         if (lockstart < lockend) {
7928                 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart,
7929                                  lockend, unlock_bits, 1, 0,
7930                                  &cached_state, GFP_NOFS);
7931         } else {
7932                 free_extent_state(cached_state);
7933         }
7934
7935         free_extent_map(em);
7936
7937         return 0;
7938
7939 unlock_err:
7940         clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7941                          unlock_bits, 1, 0, &cached_state, GFP_NOFS);
7942 err:
7943         if (dio_data)
7944                 current->journal_info = dio_data;
7945         /*
7946          * Compensate the delalloc release we do in btrfs_direct_IO() when we
7947          * write less data then expected, so that we don't underflow our inode's
7948          * outstanding extents counter.
7949          */
7950         if (create && dio_data)
7951                 adjust_dio_outstanding_extents(inode, dio_data, len);
7952
7953         return ret;
7954 }
7955
7956 static inline int submit_dio_repair_bio(struct inode *inode, struct bio *bio,
7957                                         int mirror_num)
7958 {
7959         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7960         int ret;
7961
7962         BUG_ON(bio_op(bio) == REQ_OP_WRITE);
7963
7964         bio_get(bio);
7965
7966         ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DIO_REPAIR);
7967         if (ret)
7968                 goto err;
7969
7970         ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
7971 err:
7972         bio_put(bio);
7973         return ret;
7974 }
7975
7976 static int btrfs_check_dio_repairable(struct inode *inode,
7977                                       struct bio *failed_bio,
7978                                       struct io_failure_record *failrec,
7979                                       int failed_mirror)
7980 {
7981         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7982         int num_copies;
7983
7984         num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
7985         if (num_copies == 1) {
7986                 /*
7987                  * we only have a single copy of the data, so don't bother with
7988                  * all the retry and error correction code that follows. no
7989                  * matter what the error is, it is very likely to persist.
7990                  */
7991                 btrfs_debug(fs_info,
7992                         "Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
7993                         num_copies, failrec->this_mirror, failed_mirror);
7994                 return 0;
7995         }
7996
7997         failrec->failed_mirror = failed_mirror;
7998         failrec->this_mirror++;
7999         if (failrec->this_mirror == failed_mirror)
8000                 failrec->this_mirror++;
8001
8002         if (failrec->this_mirror > num_copies) {
8003                 btrfs_debug(fs_info,
8004                         "Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
8005                         num_copies, failrec->this_mirror, failed_mirror);
8006                 return 0;
8007         }
8008
8009         return 1;
8010 }
8011
8012 static int dio_read_error(struct inode *inode, struct bio *failed_bio,
8013                         struct page *page, unsigned int pgoff,
8014                         u64 start, u64 end, int failed_mirror,
8015                         bio_end_io_t *repair_endio, void *repair_arg)
8016 {
8017         struct io_failure_record *failrec;
8018         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
8019         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
8020         struct bio *bio;
8021         int isector;
8022         unsigned int read_mode = 0;
8023         int segs;
8024         int ret;
8025
8026         BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
8027
8028         ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
8029         if (ret)
8030                 return ret;
8031
8032         ret = btrfs_check_dio_repairable(inode, failed_bio, failrec,
8033                                          failed_mirror);
8034         if (!ret) {
8035                 free_io_failure(failure_tree, io_tree, failrec);
8036                 return -EIO;
8037         }
8038
8039         segs = bio_segments(failed_bio);
8040         if (segs > 1 ||
8041             (failed_bio->bi_io_vec->bv_len > btrfs_inode_sectorsize(inode)))
8042                 read_mode |= REQ_FAILFAST_DEV;
8043
8044         isector = start - btrfs_io_bio(failed_bio)->logical;
8045         isector >>= inode->i_sb->s_blocksize_bits;
8046         bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
8047                                 pgoff, isector, repair_endio, repair_arg);
8048         bio_set_op_attrs(bio, REQ_OP_READ, read_mode);
8049
8050         btrfs_debug(BTRFS_I(inode)->root->fs_info,
8051                     "repair DIO read error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d",
8052                     read_mode, failrec->this_mirror, failrec->in_validation);
8053
8054         ret = submit_dio_repair_bio(inode, bio, failrec->this_mirror);
8055         if (ret) {
8056                 free_io_failure(failure_tree, io_tree, failrec);
8057                 bio_put(bio);
8058         }
8059
8060         return ret;
8061 }
8062
8063 struct btrfs_retry_complete {
8064         struct completion done;
8065         struct inode *inode;
8066         u64 start;
8067         int uptodate;
8068 };
8069
8070 static void btrfs_retry_endio_nocsum(struct bio *bio)
8071 {
8072         struct btrfs_retry_complete *done = bio->bi_private;
8073         struct inode *inode = done->inode;
8074         struct bio_vec *bvec;
8075         struct extent_io_tree *io_tree, *failure_tree;
8076         int i;
8077
8078         if (bio->bi_status)
8079                 goto end;
8080
8081         ASSERT(bio->bi_vcnt == 1);
8082         io_tree = &BTRFS_I(inode)->io_tree;
8083         failure_tree = &BTRFS_I(inode)->io_failure_tree;
8084         ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(inode));
8085
8086         done->uptodate = 1;
8087         ASSERT(!bio_flagged(bio, BIO_CLONED));
8088         bio_for_each_segment_all(bvec, bio, i)
8089                 clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree,
8090                                  io_tree, done->start, bvec->bv_page,
8091                                  btrfs_ino(BTRFS_I(inode)), 0);
8092 end:
8093         complete(&done->done);
8094         bio_put(bio);
8095 }
8096
8097 static int __btrfs_correct_data_nocsum(struct inode *inode,
8098                                        struct btrfs_io_bio *io_bio)
8099 {
8100         struct btrfs_fs_info *fs_info;
8101         struct bio_vec bvec;
8102         struct bvec_iter iter;
8103         struct btrfs_retry_complete done;
8104         u64 start;
8105         unsigned int pgoff;
8106         u32 sectorsize;
8107         int nr_sectors;
8108         int ret;
8109         int err = 0;
8110
8111         fs_info = BTRFS_I(inode)->root->fs_info;
8112         sectorsize = fs_info->sectorsize;
8113
8114         start = io_bio->logical;
8115         done.inode = inode;
8116         io_bio->bio.bi_iter = io_bio->iter;
8117
8118         bio_for_each_segment(bvec, &io_bio->bio, iter) {
8119                 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
8120                 pgoff = bvec.bv_offset;
8121
8122 next_block_or_try_again:
8123                 done.uptodate = 0;
8124                 done.start = start;
8125                 init_completion(&done.done);
8126
8127                 ret = dio_read_error(inode, &io_bio->bio, bvec.bv_page,
8128                                 pgoff, start, start + sectorsize - 1,
8129                                 io_bio->mirror_num,
8130                                 btrfs_retry_endio_nocsum, &done);
8131                 if (ret) {
8132                         err = ret;
8133                         goto next;
8134                 }
8135
8136                 wait_for_completion(&done.done);
8137
8138                 if (!done.uptodate) {
8139                         /* We might have another mirror, so try again */
8140                         goto next_block_or_try_again;
8141                 }
8142
8143 next:
8144                 start += sectorsize;
8145
8146                 nr_sectors--;
8147                 if (nr_sectors) {
8148                         pgoff += sectorsize;
8149                         ASSERT(pgoff < PAGE_SIZE);
8150                         goto next_block_or_try_again;
8151                 }
8152         }
8153
8154         return err;
8155 }
8156
8157 static void btrfs_retry_endio(struct bio *bio)
8158 {
8159         struct btrfs_retry_complete *done = bio->bi_private;
8160         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8161         struct extent_io_tree *io_tree, *failure_tree;
8162         struct inode *inode = done->inode;
8163         struct bio_vec *bvec;
8164         int uptodate;
8165         int ret;
8166         int i;
8167
8168         if (bio->bi_status)
8169                 goto end;
8170
8171         uptodate = 1;
8172
8173         ASSERT(bio->bi_vcnt == 1);
8174         ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(done->inode));
8175
8176         io_tree = &BTRFS_I(inode)->io_tree;
8177         failure_tree = &BTRFS_I(inode)->io_failure_tree;
8178
8179         ASSERT(!bio_flagged(bio, BIO_CLONED));
8180         bio_for_each_segment_all(bvec, bio, i) {
8181                 ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page,
8182                                              bvec->bv_offset, done->start,
8183                                              bvec->bv_len);
8184                 if (!ret)
8185                         clean_io_failure(BTRFS_I(inode)->root->fs_info,
8186                                          failure_tree, io_tree, done->start,
8187                                          bvec->bv_page,
8188                                          btrfs_ino(BTRFS_I(inode)),
8189                                          bvec->bv_offset);
8190                 else
8191                         uptodate = 0;
8192         }
8193
8194         done->uptodate = uptodate;
8195 end:
8196         complete(&done->done);
8197         bio_put(bio);
8198 }
8199
8200 static blk_status_t __btrfs_subio_endio_read(struct inode *inode,
8201                 struct btrfs_io_bio *io_bio, blk_status_t err)
8202 {
8203         struct btrfs_fs_info *fs_info;
8204         struct bio_vec bvec;
8205         struct bvec_iter iter;
8206         struct btrfs_retry_complete done;
8207         u64 start;
8208         u64 offset = 0;
8209         u32 sectorsize;
8210         int nr_sectors;
8211         unsigned int pgoff;
8212         int csum_pos;
8213         bool uptodate = (err == 0);
8214         int ret;
8215
8216         fs_info = BTRFS_I(inode)->root->fs_info;
8217         sectorsize = fs_info->sectorsize;
8218
8219         err = 0;
8220         start = io_bio->logical;
8221         done.inode = inode;
8222         io_bio->bio.bi_iter = io_bio->iter;
8223
8224         bio_for_each_segment(bvec, &io_bio->bio, iter) {
8225                 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
8226
8227                 pgoff = bvec.bv_offset;
8228 next_block:
8229                 if (uptodate) {
8230                         csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset);
8231                         ret = __readpage_endio_check(inode, io_bio, csum_pos,
8232                                         bvec.bv_page, pgoff, start, sectorsize);
8233                         if (likely(!ret))
8234                                 goto next;
8235                 }
8236 try_again:
8237                 done.uptodate = 0;
8238                 done.start = start;
8239                 init_completion(&done.done);
8240
8241                 ret = dio_read_error(inode, &io_bio->bio, bvec.bv_page,
8242                                 pgoff, start, start + sectorsize - 1,
8243                                 io_bio->mirror_num,
8244                                 btrfs_retry_endio, &done);
8245                 if (ret) {
8246                         err = errno_to_blk_status(ret);
8247                         goto next;
8248                 }
8249
8250                 wait_for_completion(&done.done);
8251
8252                 if (!done.uptodate) {
8253                         /* We might have another mirror, so try again */
8254                         goto try_again;
8255                 }
8256 next:
8257                 offset += sectorsize;
8258                 start += sectorsize;
8259
8260                 ASSERT(nr_sectors);
8261
8262                 nr_sectors--;
8263                 if (nr_sectors) {
8264                         pgoff += sectorsize;
8265                         ASSERT(pgoff < PAGE_SIZE);
8266                         goto next_block;
8267                 }
8268         }
8269
8270         return err;
8271 }
8272
8273 static blk_status_t btrfs_subio_endio_read(struct inode *inode,
8274                 struct btrfs_io_bio *io_bio, blk_status_t err)
8275 {
8276         bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
8277
8278         if (skip_csum) {
8279                 if (unlikely(err))
8280                         return __btrfs_correct_data_nocsum(inode, io_bio);
8281                 else
8282                         return 0;
8283         } else {
8284                 return __btrfs_subio_endio_read(inode, io_bio, err);
8285         }
8286 }
8287
8288 static void btrfs_endio_direct_read(struct bio *bio)
8289 {
8290         struct btrfs_dio_private *dip = bio->bi_private;
8291         struct inode *inode = dip->inode;
8292         struct bio *dio_bio;
8293         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8294         blk_status_t err = bio->bi_status;
8295
8296         if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED) {
8297                 err = btrfs_subio_endio_read(inode, io_bio, err);
8298                 if (!err)
8299                         bio->bi_status = 0;
8300         }
8301
8302         unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset,
8303                       dip->logical_offset + dip->bytes - 1);
8304         dio_bio = dip->dio_bio;
8305
8306         kfree(dip);
8307
8308         dio_bio->bi_status = bio->bi_status;
8309         dio_end_io(dio_bio);
8310
8311         if (io_bio->end_io)
8312                 io_bio->end_io(io_bio, blk_status_to_errno(err));
8313         bio_put(bio);
8314 }
8315
8316 static void __endio_write_update_ordered(struct inode *inode,
8317                                          const u64 offset, const u64 bytes,
8318                                          const bool uptodate)
8319 {
8320         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8321         struct btrfs_ordered_extent *ordered = NULL;
8322         struct btrfs_workqueue *wq;
8323         btrfs_work_func_t func;
8324         u64 ordered_offset = offset;
8325         u64 ordered_bytes = bytes;
8326         int ret;
8327
8328         if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
8329                 wq = fs_info->endio_freespace_worker;
8330                 func = btrfs_freespace_write_helper;
8331         } else {
8332                 wq = fs_info->endio_write_workers;
8333                 func = btrfs_endio_write_helper;
8334         }
8335
8336 again:
8337         ret = btrfs_dec_test_first_ordered_pending(inode, &ordered,
8338                                                    &ordered_offset,
8339                                                    ordered_bytes,
8340                                                    uptodate);
8341         if (!ret)
8342                 goto out_test;
8343
8344         btrfs_init_work(&ordered->work, func, finish_ordered_fn, NULL, NULL);
8345         btrfs_queue_work(wq, &ordered->work);
8346 out_test:
8347         /*
8348          * our bio might span multiple ordered extents.  If we haven't
8349          * completed the accounting for the whole dio, go back and try again
8350          */
8351         if (ordered_offset < offset + bytes) {
8352                 ordered_bytes = offset + bytes - ordered_offset;
8353                 ordered = NULL;
8354                 goto again;
8355         }
8356 }
8357
8358 static void btrfs_endio_direct_write(struct bio *bio)
8359 {
8360         struct btrfs_dio_private *dip = bio->bi_private;
8361         struct bio *dio_bio = dip->dio_bio;
8362
8363         __endio_write_update_ordered(dip->inode, dip->logical_offset,
8364                                      dip->bytes, !bio->bi_status);
8365
8366         kfree(dip);
8367
8368         dio_bio->bi_status = bio->bi_status;
8369         dio_end_io(dio_bio);
8370         bio_put(bio);
8371 }
8372
8373 static blk_status_t __btrfs_submit_bio_start_direct_io(void *private_data,
8374                                     struct bio *bio, int mirror_num,
8375                                     unsigned long bio_flags, u64 offset)
8376 {
8377         struct inode *inode = private_data;
8378         blk_status_t ret;
8379         ret = btrfs_csum_one_bio(inode, bio, offset, 1);
8380         BUG_ON(ret); /* -ENOMEM */
8381         return 0;
8382 }
8383
8384 static void btrfs_end_dio_bio(struct bio *bio)
8385 {
8386         struct btrfs_dio_private *dip = bio->bi_private;
8387         blk_status_t err = bio->bi_status;
8388
8389         if (err)
8390                 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info,
8391                            "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d",
8392                            btrfs_ino(BTRFS_I(dip->inode)), bio_op(bio),
8393                            bio->bi_opf,
8394                            (unsigned long long)bio->bi_iter.bi_sector,
8395                            bio->bi_iter.bi_size, err);
8396
8397         if (dip->subio_endio)
8398                 err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err);
8399
8400         if (err) {
8401                 dip->errors = 1;
8402
8403                 /*
8404                  * before atomic variable goto zero, we must make sure
8405                  * dip->errors is perceived to be set.
8406                  */
8407                 smp_mb__before_atomic();
8408         }
8409
8410         /* if there are more bios still pending for this dio, just exit */
8411         if (!atomic_dec_and_test(&dip->pending_bios))
8412                 goto out;
8413
8414         if (dip->errors) {
8415                 bio_io_error(dip->orig_bio);
8416         } else {
8417                 dip->dio_bio->bi_status = 0;
8418                 bio_endio(dip->orig_bio);
8419         }
8420 out:
8421         bio_put(bio);
8422 }
8423
8424 static inline blk_status_t btrfs_lookup_and_bind_dio_csum(struct inode *inode,
8425                                                  struct btrfs_dio_private *dip,
8426                                                  struct bio *bio,
8427                                                  u64 file_offset)
8428 {
8429         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8430         struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio);
8431         blk_status_t ret;
8432
8433         /*
8434          * We load all the csum data we need when we submit
8435          * the first bio to reduce the csum tree search and
8436          * contention.
8437          */
8438         if (dip->logical_offset == file_offset) {
8439                 ret = btrfs_lookup_bio_sums_dio(inode, dip->orig_bio,
8440                                                 file_offset);
8441                 if (ret)
8442                         return ret;
8443         }
8444
8445         if (bio == dip->orig_bio)
8446                 return 0;
8447
8448         file_offset -= dip->logical_offset;
8449         file_offset >>= inode->i_sb->s_blocksize_bits;
8450         io_bio->csum = (u8 *)(((u32 *)orig_io_bio->csum) + file_offset);
8451
8452         return 0;
8453 }
8454
8455 static inline int __btrfs_submit_dio_bio(struct bio *bio, struct inode *inode,
8456                                          u64 file_offset, int skip_sum,
8457                                          int async_submit)
8458 {
8459         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8460         struct btrfs_dio_private *dip = bio->bi_private;
8461         bool write = bio_op(bio) == REQ_OP_WRITE;
8462         blk_status_t ret;
8463
8464         if (async_submit)
8465                 async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers);
8466
8467         bio_get(bio);
8468
8469         if (!write) {
8470                 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA);
8471                 if (ret)
8472                         goto err;
8473         }
8474
8475         if (skip_sum)
8476                 goto map;
8477
8478         if (write && async_submit) {
8479                 ret = btrfs_wq_submit_bio(fs_info, bio, 0, 0,
8480                                           file_offset, inode,
8481                                           __btrfs_submit_bio_start_direct_io,
8482                                           __btrfs_submit_bio_done);
8483                 goto err;
8484         } else if (write) {
8485                 /*
8486                  * If we aren't doing async submit, calculate the csum of the
8487                  * bio now.
8488                  */
8489                 ret = btrfs_csum_one_bio(inode, bio, file_offset, 1);
8490                 if (ret)
8491                         goto err;
8492         } else {
8493                 ret = btrfs_lookup_and_bind_dio_csum(inode, dip, bio,
8494                                                      file_offset);
8495                 if (ret)
8496                         goto err;
8497         }
8498 map:
8499         ret = btrfs_map_bio(fs_info, bio, 0, async_submit);
8500 err:
8501         bio_put(bio);
8502         return ret;
8503 }
8504
8505 static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip,
8506                                     int skip_sum)
8507 {
8508         struct inode *inode = dip->inode;
8509         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8510         struct bio *bio;
8511         struct bio *orig_bio = dip->orig_bio;
8512         u64 start_sector = orig_bio->bi_iter.bi_sector;
8513         u64 file_offset = dip->logical_offset;
8514         u64 map_length;
8515         int async_submit = 0;
8516         u64 submit_len;
8517         int clone_offset = 0;
8518         int clone_len;
8519         int ret;
8520
8521         map_length = orig_bio->bi_iter.bi_size;
8522         submit_len = map_length;
8523         ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), start_sector << 9,
8524                               &map_length, NULL, 0);
8525         if (ret)
8526                 return -EIO;
8527
8528         if (map_length >= submit_len) {
8529                 bio = orig_bio;
8530                 dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED;
8531                 goto submit;
8532         }
8533
8534         /* async crcs make it difficult to collect full stripe writes. */
8535         if (btrfs_data_alloc_profile(fs_info) & BTRFS_BLOCK_GROUP_RAID56_MASK)
8536                 async_submit = 0;
8537         else
8538                 async_submit = 1;
8539
8540         /* bio split */
8541         ASSERT(map_length <= INT_MAX);
8542         atomic_inc(&dip->pending_bios);
8543         do {
8544                 clone_len = min_t(int, submit_len, map_length);
8545
8546                 /*
8547                  * This will never fail as it's passing GPF_NOFS and
8548                  * the allocation is backed by btrfs_bioset.
8549                  */
8550                 bio = btrfs_bio_clone_partial(orig_bio, clone_offset,
8551                                               clone_len);
8552                 bio->bi_private = dip;
8553                 bio->bi_end_io = btrfs_end_dio_bio;
8554                 btrfs_io_bio(bio)->logical = file_offset;
8555
8556                 ASSERT(submit_len >= clone_len);
8557                 submit_len -= clone_len;
8558                 if (submit_len == 0)
8559                         break;
8560
8561                 /*
8562                  * Increase the count before we submit the bio so we know
8563                  * the end IO handler won't happen before we increase the
8564                  * count. Otherwise, the dip might get freed before we're
8565                  * done setting it up.
8566                  */
8567                 atomic_inc(&dip->pending_bios);
8568
8569                 ret = __btrfs_submit_dio_bio(bio, inode, file_offset, skip_sum,
8570                                              async_submit);
8571                 if (ret) {
8572                         bio_put(bio);
8573                         atomic_dec(&dip->pending_bios);
8574                         goto out_err;
8575                 }
8576
8577                 clone_offset += clone_len;
8578                 start_sector += clone_len >> 9;
8579                 file_offset += clone_len;
8580
8581                 map_length = submit_len;
8582                 ret = btrfs_map_block(fs_info, btrfs_op(orig_bio),
8583                                       start_sector << 9, &map_length, NULL, 0);
8584                 if (ret)
8585                         goto out_err;
8586         } while (submit_len > 0);
8587
8588 submit:
8589         ret = __btrfs_submit_dio_bio(bio, inode, file_offset, skip_sum,
8590                                      async_submit);
8591         if (!ret)
8592                 return 0;
8593
8594         bio_put(bio);
8595 out_err:
8596         dip->errors = 1;
8597         /*
8598          * before atomic variable goto zero, we must
8599          * make sure dip->errors is perceived to be set.
8600          */
8601         smp_mb__before_atomic();
8602         if (atomic_dec_and_test(&dip->pending_bios))
8603                 bio_io_error(dip->orig_bio);
8604
8605         /* bio_end_io() will handle error, so we needn't return it */
8606         return 0;
8607 }
8608
8609 static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode,
8610                                 loff_t file_offset)
8611 {
8612         struct btrfs_dio_private *dip = NULL;
8613         struct bio *bio = NULL;
8614         struct btrfs_io_bio *io_bio;
8615         int skip_sum;
8616         bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
8617         int ret = 0;
8618
8619         skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
8620
8621         bio = btrfs_bio_clone(dio_bio);
8622
8623         dip = kzalloc(sizeof(*dip), GFP_NOFS);
8624         if (!dip) {
8625                 ret = -ENOMEM;
8626                 goto free_ordered;
8627         }
8628
8629         dip->private = dio_bio->bi_private;
8630         dip->inode = inode;
8631         dip->logical_offset = file_offset;
8632         dip->bytes = dio_bio->bi_iter.bi_size;
8633         dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
8634         bio->bi_private = dip;
8635         dip->orig_bio = bio;
8636         dip->dio_bio = dio_bio;
8637         atomic_set(&dip->pending_bios, 0);
8638         io_bio = btrfs_io_bio(bio);
8639         io_bio->logical = file_offset;
8640
8641         if (write) {
8642                 bio->bi_end_io = btrfs_endio_direct_write;
8643         } else {
8644                 bio->bi_end_io = btrfs_endio_direct_read;
8645                 dip->subio_endio = btrfs_subio_endio_read;
8646         }
8647
8648         /*
8649          * Reset the range for unsubmitted ordered extents (to a 0 length range)
8650          * even if we fail to submit a bio, because in such case we do the
8651          * corresponding error handling below and it must not be done a second
8652          * time by btrfs_direct_IO().
8653          */
8654         if (write) {
8655                 struct btrfs_dio_data *dio_data = current->journal_info;
8656
8657                 dio_data->unsubmitted_oe_range_end = dip->logical_offset +
8658                         dip->bytes;
8659                 dio_data->unsubmitted_oe_range_start =
8660                         dio_data->unsubmitted_oe_range_end;
8661         }
8662
8663         ret = btrfs_submit_direct_hook(dip, skip_sum);
8664         if (!ret)
8665                 return;
8666
8667         if (io_bio->end_io)
8668                 io_bio->end_io(io_bio, ret);
8669
8670 free_ordered:
8671         /*
8672          * If we arrived here it means either we failed to submit the dip
8673          * or we either failed to clone the dio_bio or failed to allocate the
8674          * dip. If we cloned the dio_bio and allocated the dip, we can just
8675          * call bio_endio against our io_bio so that we get proper resource
8676          * cleanup if we fail to submit the dip, otherwise, we must do the
8677          * same as btrfs_endio_direct_[write|read] because we can't call these
8678          * callbacks - they require an allocated dip and a clone of dio_bio.
8679          */
8680         if (bio && dip) {
8681                 bio_io_error(bio);
8682                 /*
8683                  * The end io callbacks free our dip, do the final put on bio
8684                  * and all the cleanup and final put for dio_bio (through
8685                  * dio_end_io()).
8686                  */
8687                 dip = NULL;
8688                 bio = NULL;
8689         } else {
8690                 if (write)
8691                         __endio_write_update_ordered(inode,
8692                                                 file_offset,
8693                                                 dio_bio->bi_iter.bi_size,
8694                                                 false);
8695                 else
8696                         unlock_extent(&BTRFS_I(inode)->io_tree, file_offset,
8697                               file_offset + dio_bio->bi_iter.bi_size - 1);
8698
8699                 dio_bio->bi_status = BLK_STS_IOERR;
8700                 /*
8701                  * Releases and cleans up our dio_bio, no need to bio_put()
8702                  * nor bio_endio()/bio_io_error() against dio_bio.
8703                  */
8704                 dio_end_io(dio_bio);
8705         }
8706         if (bio)
8707                 bio_put(bio);
8708         kfree(dip);
8709 }
8710
8711 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
8712                                struct kiocb *iocb,
8713                                const struct iov_iter *iter, loff_t offset)
8714 {
8715         int seg;
8716         int i;
8717         unsigned int blocksize_mask = fs_info->sectorsize - 1;
8718         ssize_t retval = -EINVAL;
8719
8720         if (offset & blocksize_mask)
8721                 goto out;
8722
8723         if (iov_iter_alignment(iter) & blocksize_mask)
8724                 goto out;
8725
8726         /* If this is a write we don't need to check anymore */
8727         if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter))
8728                 return 0;
8729         /*
8730          * Check to make sure we don't have duplicate iov_base's in this
8731          * iovec, if so return EINVAL, otherwise we'll get csum errors
8732          * when reading back.
8733          */
8734         for (seg = 0; seg < iter->nr_segs; seg++) {
8735                 for (i = seg + 1; i < iter->nr_segs; i++) {
8736                         if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
8737                                 goto out;
8738                 }
8739         }
8740         retval = 0;
8741 out:
8742         return retval;
8743 }
8744
8745 static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
8746 {
8747         struct file *file = iocb->ki_filp;
8748         struct inode *inode = file->f_mapping->host;
8749         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8750         struct btrfs_dio_data dio_data = { 0 };
8751         struct extent_changeset *data_reserved = NULL;
8752         loff_t offset = iocb->ki_pos;
8753         size_t count = 0;
8754         int flags = 0;
8755         bool wakeup = true;
8756         bool relock = false;
8757         ssize_t ret;
8758
8759         if (check_direct_IO(fs_info, iocb, iter, offset))
8760                 return 0;
8761
8762         inode_dio_begin(inode);
8763         smp_mb__after_atomic();
8764
8765         /*
8766          * The generic stuff only does filemap_write_and_wait_range, which
8767          * isn't enough if we've written compressed pages to this area, so
8768          * we need to flush the dirty pages again to make absolutely sure
8769          * that any outstanding dirty pages are on disk.
8770          */
8771         count = iov_iter_count(iter);
8772         if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
8773                      &BTRFS_I(inode)->runtime_flags))
8774                 filemap_fdatawrite_range(inode->i_mapping, offset,
8775                                          offset + count - 1);
8776
8777         if (iov_iter_rw(iter) == WRITE) {
8778                 /*
8779                  * If the write DIO is beyond the EOF, we need update
8780                  * the isize, but it is protected by i_mutex. So we can
8781                  * not unlock the i_mutex at this case.
8782                  */
8783                 if (offset + count <= inode->i_size) {
8784                         dio_data.overwrite = 1;
8785                         inode_unlock(inode);
8786                         relock = true;
8787                 } else if (iocb->ki_flags & IOCB_NOWAIT) {
8788                         ret = -EAGAIN;
8789                         goto out;
8790                 }
8791                 ret = btrfs_delalloc_reserve_space(inode, &data_reserved,
8792                                                    offset, count);
8793                 if (ret)
8794                         goto out;
8795                 dio_data.outstanding_extents = count_max_extents(count);
8796
8797                 /*
8798                  * We need to know how many extents we reserved so that we can
8799                  * do the accounting properly if we go over the number we
8800                  * originally calculated.  Abuse current->journal_info for this.
8801                  */
8802                 dio_data.reserve = round_up(count,
8803                                             fs_info->sectorsize);
8804                 dio_data.unsubmitted_oe_range_start = (u64)offset;
8805                 dio_data.unsubmitted_oe_range_end = (u64)offset;
8806                 current->journal_info = &dio_data;
8807                 down_read(&BTRFS_I(inode)->dio_sem);
8808         } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
8809                                      &BTRFS_I(inode)->runtime_flags)) {
8810                 inode_dio_end(inode);
8811                 flags = DIO_LOCKING | DIO_SKIP_HOLES;
8812                 wakeup = false;
8813         }
8814
8815         ret = __blockdev_direct_IO(iocb, inode,
8816                                    fs_info->fs_devices->latest_bdev,
8817                                    iter, btrfs_get_blocks_direct, NULL,
8818                                    btrfs_submit_direct, flags);
8819         if (iov_iter_rw(iter) == WRITE) {
8820                 up_read(&BTRFS_I(inode)->dio_sem);
8821                 current->journal_info = NULL;
8822                 if (ret < 0 && ret != -EIOCBQUEUED) {
8823                         if (dio_data.reserve)
8824                                 btrfs_delalloc_release_space(inode, data_reserved,
8825                                         offset, dio_data.reserve);
8826                         /*
8827                          * On error we might have left some ordered extents
8828                          * without submitting corresponding bios for them, so
8829                          * cleanup them up to avoid other tasks getting them
8830                          * and waiting for them to complete forever.
8831                          */
8832                         if (dio_data.unsubmitted_oe_range_start <
8833                             dio_data.unsubmitted_oe_range_end)
8834                                 __endio_write_update_ordered(inode,
8835                                         dio_data.unsubmitted_oe_range_start,
8836                                         dio_data.unsubmitted_oe_range_end -
8837                                         dio_data.unsubmitted_oe_range_start,
8838                                         false);
8839                 } else if (ret >= 0 && (size_t)ret < count)
8840                         btrfs_delalloc_release_space(inode, data_reserved,
8841                                         offset, count - (size_t)ret);
8842         }
8843 out:
8844         if (wakeup)
8845                 inode_dio_end(inode);
8846         if (relock)
8847                 inode_lock(inode);
8848
8849         extent_changeset_free(data_reserved);
8850         return ret;
8851 }
8852
8853 #define BTRFS_FIEMAP_FLAGS      (FIEMAP_FLAG_SYNC)
8854
8855 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
8856                 __u64 start, __u64 len)
8857 {
8858         int     ret;
8859
8860         ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS);
8861         if (ret)
8862                 return ret;
8863
8864         return extent_fiemap(inode, fieinfo, start, len, btrfs_get_extent_fiemap);
8865 }
8866
8867 int btrfs_readpage(struct file *file, struct page *page)
8868 {
8869         struct extent_io_tree *tree;
8870         tree = &BTRFS_I(page->mapping->host)->io_tree;
8871         return extent_read_full_page(tree, page, btrfs_get_extent, 0);
8872 }
8873
8874 static int btrfs_writepage(struct page *page, struct writeback_control *wbc)
8875 {
8876         struct extent_io_tree *tree;
8877         struct inode *inode = page->mapping->host;
8878         int ret;
8879
8880         if (current->flags & PF_MEMALLOC) {
8881                 redirty_page_for_writepage(wbc, page);
8882                 unlock_page(page);
8883                 return 0;
8884         }
8885
8886         /*
8887          * If we are under memory pressure we will call this directly from the
8888          * VM, we need to make sure we have the inode referenced for the ordered
8889          * extent.  If not just return like we didn't do anything.
8890          */
8891         if (!igrab(inode)) {
8892                 redirty_page_for_writepage(wbc, page);
8893                 return AOP_WRITEPAGE_ACTIVATE;
8894         }
8895         tree = &BTRFS_I(page->mapping->host)->io_tree;
8896         ret = extent_write_full_page(tree, page, btrfs_get_extent, wbc);
8897         btrfs_add_delayed_iput(inode);
8898         return ret;
8899 }
8900
8901 static int btrfs_writepages(struct address_space *mapping,
8902                             struct writeback_control *wbc)
8903 {
8904         struct extent_io_tree *tree;
8905
8906         tree = &BTRFS_I(mapping->host)->io_tree;
8907         return extent_writepages(tree, mapping, btrfs_get_extent, wbc);
8908 }
8909
8910 static int
8911 btrfs_readpages(struct file *file, struct address_space *mapping,
8912                 struct list_head *pages, unsigned nr_pages)
8913 {
8914         struct extent_io_tree *tree;
8915         tree = &BTRFS_I(mapping->host)->io_tree;
8916         return extent_readpages(tree, mapping, pages, nr_pages,
8917                                 btrfs_get_extent);
8918 }
8919 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8920 {
8921         struct extent_io_tree *tree;
8922         struct extent_map_tree *map;
8923         int ret;
8924
8925         tree = &BTRFS_I(page->mapping->host)->io_tree;
8926         map = &BTRFS_I(page->mapping->host)->extent_tree;
8927         ret = try_release_extent_mapping(map, tree, page, gfp_flags);
8928         if (ret == 1) {
8929                 ClearPagePrivate(page);
8930                 set_page_private(page, 0);
8931                 put_page(page);
8932         }
8933         return ret;
8934 }
8935
8936 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8937 {
8938         if (PageWriteback(page) || PageDirty(page))
8939                 return 0;
8940         return __btrfs_releasepage(page, gfp_flags);
8941 }
8942
8943 static void btrfs_invalidatepage(struct page *page, unsigned int offset,
8944                                  unsigned int length)
8945 {
8946         struct inode *inode = page->mapping->host;
8947         struct extent_io_tree *tree;
8948         struct btrfs_ordered_extent *ordered;
8949         struct extent_state *cached_state = NULL;
8950         u64 page_start = page_offset(page);
8951         u64 page_end = page_start + PAGE_SIZE - 1;
8952         u64 start;
8953         u64 end;
8954         int inode_evicting = inode->i_state & I_FREEING;
8955
8956         /*
8957          * we have the page locked, so new writeback can't start,
8958          * and the dirty bit won't be cleared while we are here.
8959          *
8960          * Wait for IO on this page so that we can safely clear
8961          * the PagePrivate2 bit and do ordered accounting
8962          */
8963         wait_on_page_writeback(page);
8964
8965         tree = &BTRFS_I(inode)->io_tree;
8966         if (offset) {
8967                 btrfs_releasepage(page, GFP_NOFS);
8968                 return;
8969         }
8970
8971         if (!inode_evicting)
8972                 lock_extent_bits(tree, page_start, page_end, &cached_state);
8973 again:
8974         start = page_start;
8975         ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
8976                                         page_end - start + 1);
8977         if (ordered) {
8978                 end = min(page_end, ordered->file_offset + ordered->len - 1);
8979                 /*
8980                  * IO on this page will never be started, so we need
8981                  * to account for any ordered extents now
8982                  */
8983                 if (!inode_evicting)
8984                         clear_extent_bit(tree, start, end,
8985                                          EXTENT_DIRTY | EXTENT_DELALLOC |
8986                                          EXTENT_DELALLOC_NEW |
8987                                          EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
8988                                          EXTENT_DEFRAG, 1, 0, &cached_state,
8989                                          GFP_NOFS);
8990                 /*
8991                  * whoever cleared the private bit is responsible
8992                  * for the finish_ordered_io
8993                  */
8994                 if (TestClearPagePrivate2(page)) {
8995                         struct btrfs_ordered_inode_tree *tree;
8996                         u64 new_len;
8997
8998                         tree = &BTRFS_I(inode)->ordered_tree;
8999
9000                         spin_lock_irq(&tree->lock);
9001                         set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);
9002                         new_len = start - ordered->file_offset;
9003                         if (new_len < ordered->truncated_len)
9004                                 ordered->truncated_len = new_len;
9005                         spin_unlock_irq(&tree->lock);
9006
9007                         if (btrfs_dec_test_ordered_pending(inode, &ordered,
9008                                                            start,
9009                                                            end - start + 1, 1))
9010                                 btrfs_finish_ordered_io(ordered);
9011                 }
9012                 btrfs_put_ordered_extent(ordered);
9013                 if (!inode_evicting) {
9014                         cached_state = NULL;
9015                         lock_extent_bits(tree, start, end,
9016                                          &cached_state);
9017                 }
9018
9019                 start = end + 1;
9020                 if (start < page_end)
9021                         goto again;
9022         }
9023
9024         /*
9025          * Qgroup reserved space handler
9026          * Page here will be either
9027          * 1) Already written to disk
9028          *    In this case, its reserved space is released from data rsv map
9029          *    and will be freed by delayed_ref handler finally.
9030          *    So even we call qgroup_free_data(), it won't decrease reserved
9031          *    space.
9032          * 2) Not written to disk
9033          *    This means the reserved space should be freed here. However,
9034          *    if a truncate invalidates the page (by clearing PageDirty)
9035          *    and the page is accounted for while allocating extent
9036          *    in btrfs_check_data_free_space() we let delayed_ref to
9037          *    free the entire extent.
9038          */
9039         if (PageDirty(page))
9040                 btrfs_qgroup_free_data(inode, NULL, page_start, PAGE_SIZE);
9041         if (!inode_evicting) {
9042                 clear_extent_bit(tree, page_start, page_end,
9043                                  EXTENT_LOCKED | EXTENT_DIRTY |
9044                                  EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
9045                                  EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1,
9046                                  &cached_state, GFP_NOFS);
9047
9048                 __btrfs_releasepage(page, GFP_NOFS);
9049         }
9050
9051         ClearPageChecked(page);
9052         if (PagePrivate(page)) {
9053                 ClearPagePrivate(page);
9054                 set_page_private(page, 0);
9055                 put_page(page);
9056         }
9057 }
9058
9059 /*
9060  * btrfs_page_mkwrite() is not allowed to change the file size as it gets
9061  * called from a page fault handler when a page is first dirtied. Hence we must
9062  * be careful to check for EOF conditions here. We set the page up correctly
9063  * for a written page which means we get ENOSPC checking when writing into
9064  * holes and correct delalloc and unwritten extent mapping on filesystems that
9065  * support these features.
9066  *
9067  * We are not allowed to take the i_mutex here so we have to play games to
9068  * protect against truncate races as the page could now be beyond EOF.  Because
9069  * vmtruncate() writes the inode size before removing pages, once we have the
9070  * page lock we can determine safely if the page is beyond EOF. If it is not
9071  * beyond EOF, then the page is guaranteed safe against truncation until we
9072  * unlock the page.
9073  */
9074 int btrfs_page_mkwrite(struct vm_fault *vmf)
9075 {
9076         struct page *page = vmf->page;
9077         struct inode *inode = file_inode(vmf->vma->vm_file);
9078         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9079         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
9080         struct btrfs_ordered_extent *ordered;
9081         struct extent_state *cached_state = NULL;
9082         struct extent_changeset *data_reserved = NULL;
9083         char *kaddr;
9084         unsigned long zero_start;
9085         loff_t size;
9086         int ret;
9087         int reserved = 0;
9088         u64 reserved_space;
9089         u64 page_start;
9090         u64 page_end;
9091         u64 end;
9092
9093         reserved_space = PAGE_SIZE;
9094
9095         sb_start_pagefault(inode->i_sb);
9096         page_start = page_offset(page);
9097         page_end = page_start + PAGE_SIZE - 1;
9098         end = page_end;
9099
9100         /*
9101          * Reserving delalloc space after obtaining the page lock can lead to
9102          * deadlock. For example, if a dirty page is locked by this function
9103          * and the call to btrfs_delalloc_reserve_space() ends up triggering
9104          * dirty page write out, then the btrfs_writepage() function could
9105          * end up waiting indefinitely to get a lock on the page currently
9106          * being processed by btrfs_page_mkwrite() function.
9107          */
9108         ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
9109                                            reserved_space);
9110         if (!ret) {
9111                 ret = file_update_time(vmf->vma->vm_file);
9112                 reserved = 1;
9113         }
9114         if (ret) {
9115                 if (ret == -ENOMEM)
9116                         ret = VM_FAULT_OOM;
9117                 else /* -ENOSPC, -EIO, etc */
9118                         ret = VM_FAULT_SIGBUS;
9119                 if (reserved)
9120                         goto out;
9121                 goto out_noreserve;
9122         }
9123
9124         ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
9125 again:
9126         lock_page(page);
9127         size = i_size_read(inode);
9128
9129         if ((page->mapping != inode->i_mapping) ||
9130             (page_start >= size)) {
9131                 /* page got truncated out from underneath us */
9132                 goto out_unlock;
9133         }
9134         wait_on_page_writeback(page);
9135
9136         lock_extent_bits(io_tree, page_start, page_end, &cached_state);
9137         set_page_extent_mapped(page);
9138
9139         /*
9140          * we can't set the delalloc bits if there are pending ordered
9141          * extents.  Drop our locks and wait for them to finish
9142          */
9143         ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
9144                         PAGE_SIZE);
9145         if (ordered) {
9146                 unlock_extent_cached(io_tree, page_start, page_end,
9147                                      &cached_state, GFP_NOFS);
9148                 unlock_page(page);
9149                 btrfs_start_ordered_extent(inode, ordered, 1);
9150                 btrfs_put_ordered_extent(ordered);
9151                 goto again;
9152         }
9153
9154         if (page->index == ((size - 1) >> PAGE_SHIFT)) {
9155                 reserved_space = round_up(size - page_start,
9156                                           fs_info->sectorsize);
9157                 if (reserved_space < PAGE_SIZE) {
9158                         end = page_start + reserved_space - 1;
9159                         spin_lock(&BTRFS_I(inode)->lock);
9160                         BTRFS_I(inode)->outstanding_extents++;
9161                         spin_unlock(&BTRFS_I(inode)->lock);
9162                         btrfs_delalloc_release_space(inode, data_reserved,
9163                                         page_start, PAGE_SIZE - reserved_space);
9164                 }
9165         }
9166
9167         /*
9168          * page_mkwrite gets called when the page is firstly dirtied after it's
9169          * faulted in, but write(2) could also dirty a page and set delalloc
9170          * bits, thus in this case for space account reason, we still need to
9171          * clear any delalloc bits within this page range since we have to
9172          * reserve data&meta space before lock_page() (see above comments).
9173          */
9174         clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end,
9175                           EXTENT_DIRTY | EXTENT_DELALLOC |
9176                           EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
9177                           0, 0, &cached_state, GFP_NOFS);
9178
9179         ret = btrfs_set_extent_delalloc(inode, page_start, end,
9180                                         &cached_state, 0);
9181         if (ret) {
9182                 unlock_extent_cached(io_tree, page_start, page_end,
9183                                      &cached_state, GFP_NOFS);
9184                 ret = VM_FAULT_SIGBUS;
9185                 goto out_unlock;
9186         }
9187         ret = 0;
9188
9189         /* page is wholly or partially inside EOF */
9190         if (page_start + PAGE_SIZE > size)
9191                 zero_start = size & ~PAGE_MASK;
9192         else
9193                 zero_start = PAGE_SIZE;
9194
9195         if (zero_start != PAGE_SIZE) {
9196                 kaddr = kmap(page);
9197                 memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start);
9198                 flush_dcache_page(page);
9199                 kunmap(page);
9200         }
9201         ClearPageChecked(page);
9202         set_page_dirty(page);
9203         SetPageUptodate(page);
9204
9205         BTRFS_I(inode)->last_trans = fs_info->generation;
9206         BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid;
9207         BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit;
9208
9209         unlock_extent_cached(io_tree, page_start, page_end, &cached_state, GFP_NOFS);
9210
9211 out_unlock:
9212         if (!ret) {
9213                 sb_end_pagefault(inode->i_sb);
9214                 extent_changeset_free(data_reserved);
9215                 return VM_FAULT_LOCKED;
9216         }
9217         unlock_page(page);
9218 out:
9219         btrfs_delalloc_release_space(inode, data_reserved, page_start,
9220                                      reserved_space);
9221 out_noreserve:
9222         sb_end_pagefault(inode->i_sb);
9223         extent_changeset_free(data_reserved);
9224         return ret;
9225 }
9226
9227 static int btrfs_truncate(struct inode *inode)
9228 {
9229         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9230         struct btrfs_root *root = BTRFS_I(inode)->root;
9231         struct btrfs_block_rsv *rsv;
9232         int ret = 0;
9233         int err = 0;
9234         struct btrfs_trans_handle *trans;
9235         u64 mask = fs_info->sectorsize - 1;
9236         u64 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1);
9237
9238         ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask),
9239                                        (u64)-1);
9240         if (ret)
9241                 return ret;
9242
9243         /*
9244          * Yes ladies and gentlemen, this is indeed ugly.  The fact is we have
9245          * 3 things going on here
9246          *
9247          * 1) We need to reserve space for our orphan item and the space to
9248          * delete our orphan item.  Lord knows we don't want to have a dangling
9249          * orphan item because we didn't reserve space to remove it.
9250          *
9251          * 2) We need to reserve space to update our inode.
9252          *
9253          * 3) We need to have something to cache all the space that is going to
9254          * be free'd up by the truncate operation, but also have some slack
9255          * space reserved in case it uses space during the truncate (thank you
9256          * very much snapshotting).
9257          *
9258          * And we need these to all be separate.  The fact is we can use a lot of
9259          * space doing the truncate, and we have no earthly idea how much space
9260          * we will use, so we need the truncate reservation to be separate so it
9261          * doesn't end up using space reserved for updating the inode or
9262          * removing the orphan item.  We also need to be able to stop the
9263          * transaction and start a new one, which means we need to be able to
9264          * update the inode several times, and we have no idea of knowing how
9265          * many times that will be, so we can't just reserve 1 item for the
9266          * entirety of the operation, so that has to be done separately as well.
9267          * Then there is the orphan item, which does indeed need to be held on
9268          * to for the whole operation, and we need nobody to touch this reserved
9269          * space except the orphan code.
9270          *
9271          * So that leaves us with
9272          *
9273          * 1) root->orphan_block_rsv - for the orphan deletion.
9274          * 2) rsv - for the truncate reservation, which we will steal from the
9275          * transaction reservation.
9276          * 3) fs_info->trans_block_rsv - this will have 1 items worth left for
9277          * updating the inode.
9278          */
9279         rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
9280         if (!rsv)
9281                 return -ENOMEM;
9282         rsv->size = min_size;
9283         rsv->failfast = 1;
9284
9285         /*
9286          * 1 for the truncate slack space
9287          * 1 for updating the inode.
9288          */
9289         trans = btrfs_start_transaction(root, 2);
9290         if (IS_ERR(trans)) {
9291                 err = PTR_ERR(trans);
9292                 goto out;
9293         }
9294
9295         /* Migrate the slack space for the truncate to our reserve */
9296         ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
9297                                       min_size, 0);
9298         BUG_ON(ret);
9299
9300         /*
9301          * So if we truncate and then write and fsync we normally would just
9302          * write the extents that changed, which is a problem if we need to
9303          * first truncate that entire inode.  So set this flag so we write out
9304          * all of the extents in the inode to the sync log so we're completely
9305          * safe.
9306          */
9307         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
9308         trans->block_rsv = rsv;
9309
9310         while (1) {
9311                 ret = btrfs_truncate_inode_items(trans, root, inode,
9312                                                  inode->i_size,
9313                                                  BTRFS_EXTENT_DATA_KEY);
9314                 if (ret != -ENOSPC && ret != -EAGAIN) {
9315                         err = ret;
9316                         break;
9317                 }
9318
9319                 trans->block_rsv = &fs_info->trans_block_rsv;
9320                 ret = btrfs_update_inode(trans, root, inode);
9321                 if (ret) {
9322                         err = ret;
9323                         break;
9324                 }
9325
9326                 btrfs_end_transaction(trans);
9327                 btrfs_btree_balance_dirty(fs_info);
9328
9329                 trans = btrfs_start_transaction(root, 2);
9330                 if (IS_ERR(trans)) {
9331                         ret = err = PTR_ERR(trans);
9332                         trans = NULL;
9333                         break;
9334                 }
9335
9336                 btrfs_block_rsv_release(fs_info, rsv, -1);
9337                 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
9338                                               rsv, min_size, 0);
9339                 BUG_ON(ret);    /* shouldn't happen */
9340                 trans->block_rsv = rsv;
9341         }
9342
9343         if (ret == 0 && inode->i_nlink > 0) {
9344                 trans->block_rsv = root->orphan_block_rsv;
9345                 ret = btrfs_orphan_del(trans, BTRFS_I(inode));
9346                 if (ret)
9347                         err = ret;
9348         }
9349
9350         if (trans) {
9351                 trans->block_rsv = &fs_info->trans_block_rsv;
9352                 ret = btrfs_update_inode(trans, root, inode);
9353                 if (ret && !err)
9354                         err = ret;
9355
9356                 ret = btrfs_end_transaction(trans);
9357                 btrfs_btree_balance_dirty(fs_info);
9358         }
9359 out:
9360         btrfs_free_block_rsv(fs_info, rsv);
9361
9362         if (ret && !err)
9363                 err = ret;
9364
9365         return err;
9366 }
9367
9368 /*
9369  * create a new subvolume directory/inode (helper for the ioctl).
9370  */
9371 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans,
9372                              struct btrfs_root *new_root,
9373                              struct btrfs_root *parent_root,
9374                              u64 new_dirid)
9375 {
9376         struct inode *inode;
9377         int err;
9378         u64 index = 0;
9379
9380         inode = btrfs_new_inode(trans, new_root, NULL, "..", 2,
9381                                 new_dirid, new_dirid,
9382                                 S_IFDIR | (~current_umask() & S_IRWXUGO),
9383                                 &index);
9384         if (IS_ERR(inode))
9385                 return PTR_ERR(inode);
9386         inode->i_op = &btrfs_dir_inode_operations;
9387         inode->i_fop = &btrfs_dir_file_operations;
9388
9389         set_nlink(inode, 1);
9390         btrfs_i_size_write(BTRFS_I(inode), 0);
9391         unlock_new_inode(inode);
9392
9393         err = btrfs_subvol_inherit_props(trans, new_root, parent_root);
9394         if (err)
9395                 btrfs_err(new_root->fs_info,
9396                           "error inheriting subvolume %llu properties: %d",
9397                           new_root->root_key.objectid, err);
9398
9399         err = btrfs_update_inode(trans, new_root, inode);
9400
9401         iput(inode);
9402         return err;
9403 }
9404
9405 struct inode *btrfs_alloc_inode(struct super_block *sb)
9406 {
9407         struct btrfs_inode *ei;
9408         struct inode *inode;
9409
9410         ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_NOFS);
9411         if (!ei)
9412                 return NULL;
9413
9414         ei->root = NULL;
9415         ei->generation = 0;
9416         ei->last_trans = 0;
9417         ei->last_sub_trans = 0;
9418         ei->logged_trans = 0;
9419         ei->delalloc_bytes = 0;
9420         ei->new_delalloc_bytes = 0;
9421         ei->defrag_bytes = 0;
9422         ei->disk_i_size = 0;
9423         ei->flags = 0;
9424         ei->csum_bytes = 0;
9425         ei->index_cnt = (u64)-1;
9426         ei->dir_index = 0;
9427         ei->last_unlink_trans = 0;
9428         ei->last_log_commit = 0;
9429         ei->delayed_iput_count = 0;
9430
9431         spin_lock_init(&ei->lock);
9432         ei->outstanding_extents = 0;
9433         ei->reserved_extents = 0;
9434
9435         ei->runtime_flags = 0;
9436         ei->force_compress = BTRFS_COMPRESS_NONE;
9437
9438         ei->delayed_node = NULL;
9439
9440         ei->i_otime.tv_sec = 0;
9441         ei->i_otime.tv_nsec = 0;
9442
9443         inode = &ei->vfs_inode;
9444         extent_map_tree_init(&ei->extent_tree);
9445         extent_io_tree_init(&ei->io_tree, inode);
9446         extent_io_tree_init(&ei->io_failure_tree, inode);
9447         ei->io_tree.track_uptodate = 1;
9448         ei->io_failure_tree.track_uptodate = 1;
9449         atomic_set(&ei->sync_writers, 0);
9450         mutex_init(&ei->log_mutex);
9451         mutex_init(&ei->delalloc_mutex);
9452         btrfs_ordered_inode_tree_init(&ei->ordered_tree);
9453         INIT_LIST_HEAD(&ei->delalloc_inodes);
9454         INIT_LIST_HEAD(&ei->delayed_iput);
9455         RB_CLEAR_NODE(&ei->rb_node);
9456         init_rwsem(&ei->dio_sem);
9457
9458         return inode;
9459 }
9460
9461 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
9462 void btrfs_test_destroy_inode(struct inode *inode)
9463 {
9464         btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
9465         kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
9466 }
9467 #endif
9468
9469 static void btrfs_i_callback(struct rcu_head *head)
9470 {
9471         struct inode *inode = container_of(head, struct inode, i_rcu);
9472         kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
9473 }
9474
9475 void btrfs_destroy_inode(struct inode *inode)
9476 {
9477         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9478         struct btrfs_ordered_extent *ordered;
9479         struct btrfs_root *root = BTRFS_I(inode)->root;
9480
9481         WARN_ON(!hlist_empty(&inode->i_dentry));
9482         WARN_ON(inode->i_data.nrpages);
9483         WARN_ON(BTRFS_I(inode)->outstanding_extents);
9484         WARN_ON(BTRFS_I(inode)->reserved_extents);
9485         WARN_ON(BTRFS_I(inode)->delalloc_bytes);
9486         WARN_ON(BTRFS_I(inode)->new_delalloc_bytes);
9487         WARN_ON(BTRFS_I(inode)->csum_bytes);
9488         WARN_ON(BTRFS_I(inode)->defrag_bytes);
9489
9490         /*
9491          * This can happen where we create an inode, but somebody else also
9492          * created the same inode and we need to destroy the one we already
9493          * created.
9494          */
9495         if (!root)
9496                 goto free;
9497
9498         if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
9499                      &BTRFS_I(inode)->runtime_flags)) {
9500                 btrfs_info(fs_info, "inode %llu still on the orphan list",
9501                            btrfs_ino(BTRFS_I(inode)));
9502                 atomic_dec(&root->orphan_inodes);
9503         }
9504
9505         while (1) {
9506                 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);
9507                 if (!ordered)
9508                         break;
9509                 else {
9510                         btrfs_err(fs_info,
9511                                   "found ordered extent %llu %llu on inode cleanup",
9512                                   ordered->file_offset, ordered->len);
9513                         btrfs_remove_ordered_extent(inode, ordered);
9514                         btrfs_put_ordered_extent(ordered);
9515                         btrfs_put_ordered_extent(ordered);
9516                 }
9517         }
9518         btrfs_qgroup_check_reserved_leak(inode);
9519         inode_tree_del(inode);
9520         btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
9521 free:
9522         call_rcu(&inode->i_rcu, btrfs_i_callback);
9523 }
9524
9525 int btrfs_drop_inode(struct inode *inode)
9526 {
9527         struct btrfs_root *root = BTRFS_I(inode)->root;
9528
9529         if (root == NULL)
9530                 return 1;
9531
9532         /* the snap/subvol tree is on deleting */
9533         if (btrfs_root_refs(&root->root_item) == 0)
9534                 return 1;
9535         else
9536                 return generic_drop_inode(inode);
9537 }
9538
9539 static void init_once(void *foo)
9540 {
9541         struct btrfs_inode *ei = (struct btrfs_inode *) foo;
9542
9543         inode_init_once(&ei->vfs_inode);
9544 }
9545
9546 void btrfs_destroy_cachep(void)
9547 {
9548         /*
9549          * Make sure all delayed rcu free inodes are flushed before we
9550          * destroy cache.
9551          */
9552         rcu_barrier();
9553         kmem_cache_destroy(btrfs_inode_cachep);
9554         kmem_cache_destroy(btrfs_trans_handle_cachep);
9555         kmem_cache_destroy(btrfs_path_cachep);
9556         kmem_cache_destroy(btrfs_free_space_cachep);
9557 }
9558
9559 int btrfs_init_cachep(void)
9560 {
9561         btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
9562                         sizeof(struct btrfs_inode), 0,
9563                         SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT,
9564                         init_once);
9565         if (!btrfs_inode_cachep)
9566                 goto fail;
9567
9568         btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle",
9569                         sizeof(struct btrfs_trans_handle), 0,
9570                         SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL);
9571         if (!btrfs_trans_handle_cachep)
9572                 goto fail;
9573
9574         btrfs_path_cachep = kmem_cache_create("btrfs_path",
9575                         sizeof(struct btrfs_path), 0,
9576                         SLAB_MEM_SPREAD, NULL);
9577         if (!btrfs_path_cachep)
9578                 goto fail;
9579
9580         btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space",
9581                         sizeof(struct btrfs_free_space), 0,
9582                         SLAB_MEM_SPREAD, NULL);
9583         if (!btrfs_free_space_cachep)
9584                 goto fail;
9585
9586         return 0;
9587 fail:
9588         btrfs_destroy_cachep();
9589         return -ENOMEM;
9590 }
9591
9592 static int btrfs_getattr(const struct path *path, struct kstat *stat,
9593                          u32 request_mask, unsigned int flags)
9594 {
9595         u64 delalloc_bytes;
9596         struct inode *inode = d_inode(path->dentry);
9597         u32 blocksize = inode->i_sb->s_blocksize;
9598         u32 bi_flags = BTRFS_I(inode)->flags;
9599
9600         stat->result_mask |= STATX_BTIME;
9601         stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec;
9602         stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec;
9603         if (bi_flags & BTRFS_INODE_APPEND)
9604                 stat->attributes |= STATX_ATTR_APPEND;
9605         if (bi_flags & BTRFS_INODE_COMPRESS)
9606                 stat->attributes |= STATX_ATTR_COMPRESSED;
9607         if (bi_flags & BTRFS_INODE_IMMUTABLE)
9608                 stat->attributes |= STATX_ATTR_IMMUTABLE;
9609         if (bi_flags & BTRFS_INODE_NODUMP)
9610                 stat->attributes |= STATX_ATTR_NODUMP;
9611
9612         stat->attributes_mask |= (STATX_ATTR_APPEND |
9613                                   STATX_ATTR_COMPRESSED |
9614                                   STATX_ATTR_IMMUTABLE |
9615                                   STATX_ATTR_NODUMP);
9616
9617         generic_fillattr(inode, stat);
9618         stat->dev = BTRFS_I(inode)->root->anon_dev;
9619
9620         spin_lock(&BTRFS_I(inode)->lock);
9621         delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes;
9622         spin_unlock(&BTRFS_I(inode)->lock);
9623         stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) +
9624                         ALIGN(delalloc_bytes, blocksize)) >> 9;
9625         return 0;
9626 }
9627
9628 static int btrfs_rename_exchange(struct inode *old_dir,
9629                               struct dentry *old_dentry,
9630                               struct inode *new_dir,
9631                               struct dentry *new_dentry)
9632 {
9633         struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
9634         struct btrfs_trans_handle *trans;
9635         struct btrfs_root *root = BTRFS_I(old_dir)->root;
9636         struct btrfs_root *dest = BTRFS_I(new_dir)->root;
9637         struct inode *new_inode = new_dentry->d_inode;
9638         struct inode *old_inode = old_dentry->d_inode;
9639         struct timespec ctime = current_time(old_inode);
9640         struct dentry *parent;
9641         u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
9642         u64 new_ino = btrfs_ino(BTRFS_I(new_inode));
9643         u64 old_idx = 0;
9644         u64 new_idx = 0;
9645         u64 root_objectid;
9646         int ret;
9647         bool root_log_pinned = false;
9648         bool dest_log_pinned = false;
9649
9650         /* we only allow rename subvolume link between subvolumes */
9651         if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
9652                 return -EXDEV;
9653
9654         /* close the race window with snapshot create/destroy ioctl */
9655         if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9656                 down_read(&fs_info->subvol_sem);
9657         if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
9658                 down_read(&fs_info->subvol_sem);
9659
9660         /*
9661          * We want to reserve the absolute worst case amount of items.  So if
9662          * both inodes are subvols and we need to unlink them then that would
9663          * require 4 item modifications, but if they are both normal inodes it
9664          * would require 5 item modifications, so we'll assume their normal
9665          * inodes.  So 5 * 2 is 10, plus 2 for the new links, so 12 total items
9666          * should cover the worst case number of items we'll modify.
9667          */
9668         trans = btrfs_start_transaction(root, 12);
9669         if (IS_ERR(trans)) {
9670                 ret = PTR_ERR(trans);
9671                 goto out_notrans;
9672         }
9673
9674         /*
9675          * We need to find a free sequence number both in the source and
9676          * in the destination directory for the exchange.
9677          */
9678         ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx);
9679         if (ret)
9680                 goto out_fail;
9681         ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx);
9682         if (ret)
9683                 goto out_fail;
9684
9685         BTRFS_I(old_inode)->dir_index = 0ULL;
9686         BTRFS_I(new_inode)->dir_index = 0ULL;
9687
9688         /* Reference for the source. */
9689         if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
9690                 /* force full log commit if subvolume involved. */
9691                 btrfs_set_log_full_commit(fs_info, trans);
9692         } else {
9693                 btrfs_pin_log_trans(root);
9694                 root_log_pinned = true;
9695                 ret = btrfs_insert_inode_ref(trans, dest,
9696                                              new_dentry->d_name.name,
9697                                              new_dentry->d_name.len,
9698                                              old_ino,
9699                                              btrfs_ino(BTRFS_I(new_dir)),
9700                                              old_idx);
9701                 if (ret)
9702                         goto out_fail;
9703         }
9704
9705         /* And now for the dest. */
9706         if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
9707                 /* force full log commit if subvolume involved. */
9708                 btrfs_set_log_full_commit(fs_info, trans);
9709         } else {
9710                 btrfs_pin_log_trans(dest);
9711                 dest_log_pinned = true;
9712                 ret = btrfs_insert_inode_ref(trans, root,
9713                                              old_dentry->d_name.name,
9714                                              old_dentry->d_name.len,
9715                                              new_ino,
9716                                              btrfs_ino(BTRFS_I(old_dir)),
9717                                              new_idx);
9718                 if (ret)
9719                         goto out_fail;
9720         }
9721
9722         /* Update inode version and ctime/mtime. */
9723         inode_inc_iversion(old_dir);
9724         inode_inc_iversion(new_dir);
9725         inode_inc_iversion(old_inode);
9726         inode_inc_iversion(new_inode);
9727         old_dir->i_ctime = old_dir->i_mtime = ctime;
9728         new_dir->i_ctime = new_dir->i_mtime = ctime;
9729         old_inode->i_ctime = ctime;
9730         new_inode->i_ctime = ctime;
9731
9732         if (old_dentry->d_parent != new_dentry->d_parent) {
9733                 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
9734                                 BTRFS_I(old_inode), 1);
9735                 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir),
9736                                 BTRFS_I(new_inode), 1);
9737         }
9738
9739         /* src is a subvolume */
9740         if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
9741                 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
9742                 ret = btrfs_unlink_subvol(trans, root, old_dir,
9743                                           root_objectid,
9744                                           old_dentry->d_name.name,
9745                                           old_dentry->d_name.len);
9746         } else { /* src is an inode */
9747                 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir),
9748                                            BTRFS_I(old_dentry->d_inode),
9749                                            old_dentry->d_name.name,
9750                                            old_dentry->d_name.len);
9751                 if (!ret)
9752                         ret = btrfs_update_inode(trans, root, old_inode);
9753         }
9754         if (ret) {
9755                 btrfs_abort_transaction(trans, ret);
9756                 goto out_fail;
9757         }
9758
9759         /* dest is a subvolume */
9760         if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
9761                 root_objectid = BTRFS_I(new_inode)->root->root_key.objectid;
9762                 ret = btrfs_unlink_subvol(trans, dest, new_dir,
9763                                           root_objectid,
9764                                           new_dentry->d_name.name,
9765                                           new_dentry->d_name.len);
9766         } else { /* dest is an inode */
9767                 ret = __btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir),
9768                                            BTRFS_I(new_dentry->d_inode),
9769                                            new_dentry->d_name.name,
9770                                            new_dentry->d_name.len);
9771                 if (!ret)
9772                         ret = btrfs_update_inode(trans, dest, new_inode);
9773         }
9774         if (ret) {
9775                 btrfs_abort_transaction(trans, ret);
9776                 goto out_fail;
9777         }
9778
9779         ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
9780                              new_dentry->d_name.name,
9781                              new_dentry->d_name.len, 0, old_idx);
9782         if (ret) {
9783                 btrfs_abort_transaction(trans, ret);
9784                 goto out_fail;
9785         }
9786
9787         ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),
9788                              old_dentry->d_name.name,
9789                              old_dentry->d_name.len, 0, new_idx);
9790         if (ret) {
9791                 btrfs_abort_transaction(trans, ret);
9792                 goto out_fail;
9793         }
9794
9795         if (old_inode->i_nlink == 1)
9796                 BTRFS_I(old_inode)->dir_index = old_idx;
9797         if (new_inode->i_nlink == 1)
9798                 BTRFS_I(new_inode)->dir_index = new_idx;
9799
9800         if (root_log_pinned) {
9801                 parent = new_dentry->d_parent;
9802                 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
9803                                 parent);
9804                 btrfs_end_log_trans(root);
9805                 root_log_pinned = false;
9806         }
9807         if (dest_log_pinned) {
9808                 parent = old_dentry->d_parent;
9809                 btrfs_log_new_name(trans, BTRFS_I(new_inode), BTRFS_I(new_dir),
9810                                 parent);
9811                 btrfs_end_log_trans(dest);
9812                 dest_log_pinned = false;
9813         }
9814 out_fail:
9815         /*
9816          * If we have pinned a log and an error happened, we unpin tasks
9817          * trying to sync the log and force them to fallback to a transaction
9818          * commit if the log currently contains any of the inodes involved in
9819          * this rename operation (to ensure we do not persist a log with an
9820          * inconsistent state for any of these inodes or leading to any
9821          * inconsistencies when replayed). If the transaction was aborted, the
9822          * abortion reason is propagated to userspace when attempting to commit
9823          * the transaction. If the log does not contain any of these inodes, we
9824          * allow the tasks to sync it.
9825          */
9826         if (ret && (root_log_pinned || dest_log_pinned)) {
9827                 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) ||
9828                     btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) ||
9829                     btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) ||
9830                     (new_inode &&
9831                      btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation)))
9832                         btrfs_set_log_full_commit(fs_info, trans);
9833
9834                 if (root_log_pinned) {
9835                         btrfs_end_log_trans(root);
9836                         root_log_pinned = false;
9837                 }
9838                 if (dest_log_pinned) {
9839                         btrfs_end_log_trans(dest);
9840                         dest_log_pinned = false;
9841                 }
9842         }
9843         ret = btrfs_end_transaction(trans);
9844 out_notrans:
9845         if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
9846                 up_read(&fs_info->subvol_sem);
9847         if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9848                 up_read(&fs_info->subvol_sem);
9849
9850         return ret;
9851 }
9852
9853 static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans,
9854                                      struct btrfs_root *root,
9855                                      struct inode *dir,
9856                                      struct dentry *dentry)
9857 {
9858         int ret;
9859         struct inode *inode;
9860         u64 objectid;
9861         u64 index;
9862
9863         ret = btrfs_find_free_ino(root, &objectid);
9864         if (ret)
9865                 return ret;
9866
9867         inode = btrfs_new_inode(trans, root, dir,
9868                                 dentry->d_name.name,
9869                                 dentry->d_name.len,
9870                                 btrfs_ino(BTRFS_I(dir)),
9871                                 objectid,
9872                                 S_IFCHR | WHITEOUT_MODE,
9873                                 &index);
9874
9875         if (IS_ERR(inode)) {
9876                 ret = PTR_ERR(inode);
9877                 return ret;
9878         }
9879
9880         inode->i_op = &btrfs_special_inode_operations;
9881         init_special_inode(inode, inode->i_mode,
9882                 WHITEOUT_DEV);
9883
9884         ret = btrfs_init_inode_security(trans, inode, dir,
9885                                 &dentry->d_name);
9886         if (ret)
9887                 goto out;
9888
9889         ret = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
9890                                 BTRFS_I(inode), 0, index);
9891         if (ret)
9892                 goto out;
9893
9894         ret = btrfs_update_inode(trans, root, inode);
9895 out:
9896         unlock_new_inode(inode);
9897         if (ret)
9898                 inode_dec_link_count(inode);
9899         iput(inode);
9900
9901         return ret;
9902 }
9903
9904 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry,
9905                            struct inode *new_dir, struct dentry *new_dentry,
9906                            unsigned int flags)
9907 {
9908         struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
9909         struct btrfs_trans_handle *trans;
9910         unsigned int trans_num_items;
9911         struct btrfs_root *root = BTRFS_I(old_dir)->root;
9912         struct btrfs_root *dest = BTRFS_I(new_dir)->root;
9913         struct inode *new_inode = d_inode(new_dentry);
9914         struct inode *old_inode = d_inode(old_dentry);
9915         u64 index = 0;
9916         u64 root_objectid;
9917         int ret;
9918         u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
9919         bool log_pinned = false;
9920
9921         if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
9922                 return -EPERM;
9923
9924         /* we only allow rename subvolume link between subvolumes */
9925         if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
9926                 return -EXDEV;
9927
9928         if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID ||
9929             (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID))
9930                 return -ENOTEMPTY;
9931
9932         if (S_ISDIR(old_inode->i_mode) && new_inode &&
9933             new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)
9934                 return -ENOTEMPTY;
9935
9936
9937         /* check for collisions, even if the  name isn't there */
9938         ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino,
9939                              new_dentry->d_name.name,
9940                              new_dentry->d_name.len);
9941
9942         if (ret) {
9943                 if (ret == -EEXIST) {
9944                         /* we shouldn't get
9945                          * eexist without a new_inode */
9946                         if (WARN_ON(!new_inode)) {
9947                                 return ret;
9948                         }
9949                 } else {
9950                         /* maybe -EOVERFLOW */
9951                         return ret;
9952                 }
9953         }
9954         ret = 0;
9955
9956         /*
9957          * we're using rename to replace one file with another.  Start IO on it
9958          * now so  we don't add too much work to the end of the transaction
9959          */
9960         if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)
9961                 filemap_flush(old_inode->i_mapping);
9962
9963         /* close the racy window with snapshot create/destroy ioctl */
9964         if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9965                 down_read(&fs_info->subvol_sem);
9966         /*
9967          * We want to reserve the absolute worst case amount of items.  So if
9968          * both inodes are subvols and we need to unlink them then that would
9969          * require 4 item modifications, but if they are both normal inodes it
9970          * would require 5 item modifications, so we'll assume they are normal
9971          * inodes.  So 5 * 2 is 10, plus 1 for the new link, so 11 total items
9972          * should cover the worst case number of items we'll modify.
9973          * If our rename has the whiteout flag, we need more 5 units for the
9974          * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item
9975          * when selinux is enabled).
9976          */
9977         trans_num_items = 11;
9978         if (flags & RENAME_WHITEOUT)
9979                 trans_num_items += 5;
9980         trans = btrfs_start_transaction(root, trans_num_items);
9981         if (IS_ERR(trans)) {
9982                 ret = PTR_ERR(trans);
9983                 goto out_notrans;
9984         }
9985
9986         if (dest != root)
9987                 btrfs_record_root_in_trans(trans, dest);
9988
9989         ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index);
9990         if (ret)
9991                 goto out_fail;
9992
9993         BTRFS_I(old_inode)->dir_index = 0ULL;
9994         if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
9995                 /* force full log commit if subvolume involved. */
9996                 btrfs_set_log_full_commit(fs_info, trans);
9997         } else {
9998                 btrfs_pin_log_trans(root);
9999                 log_pinned = true;
10000                 ret = btrfs_insert_inode_ref(trans, dest,
10001                                              new_dentry->d_name.name,
10002                                              new_dentry->d_name.len,
10003                                              old_ino,
10004                                              btrfs_ino(BTRFS_I(new_dir)), index);
10005                 if (ret)
10006                         goto out_fail;
10007         }
10008
10009         inode_inc_iversion(old_dir);
10010         inode_inc_iversion(new_dir);
10011         inode_inc_iversion(old_inode);
10012         old_dir->i_ctime = old_dir->i_mtime =
10013         new_dir->i_ctime = new_dir->i_mtime =
10014         old_inode->i_ctime = current_time(old_dir);
10015
10016         if (old_dentry->d_parent != new_dentry->d_parent)
10017                 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
10018                                 BTRFS_I(old_inode), 1);
10019
10020         if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
10021                 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
10022                 ret = btrfs_unlink_subvol(trans, root, old_dir, root_objectid,
10023                                         old_dentry->d_name.name,
10024                                         old_dentry->d_name.len);
10025         } else {
10026                 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir),
10027                                         BTRFS_I(d_inode(old_dentry)),
10028                                         old_dentry->d_name.name,
10029                                         old_dentry->d_name.len);
10030                 if (!ret)
10031                         ret = btrfs_update_inode(trans, root, old_inode);
10032         }
10033         if (ret) {
10034                 btrfs_abort_transaction(trans, ret);
10035                 goto out_fail;
10036         }
10037
10038         if (new_inode) {
10039                 inode_inc_iversion(new_inode);
10040                 new_inode->i_ctime = current_time(new_inode);
10041                 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) ==
10042                              BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
10043                         root_objectid = BTRFS_I(new_inode)->location.objectid;
10044                         ret = btrfs_unlink_subvol(trans, dest, new_dir,
10045                                                 root_objectid,
10046                                                 new_dentry->d_name.name,
10047                                                 new_dentry->d_name.len);
10048                         BUG_ON(new_inode->i_nlink == 0);
10049                 } else {
10050                         ret = btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir),
10051                                                  BTRFS_I(d_inode(new_dentry)),
10052                                                  new_dentry->d_name.name,
10053                                                  new_dentry->d_name.len);
10054                 }
10055                 if (!ret && new_inode->i_nlink == 0)
10056                         ret = btrfs_orphan_add(trans,
10057                                         BTRFS_I(d_inode(new_dentry)));
10058                 if (ret) {
10059                         btrfs_abort_transaction(trans, ret);
10060                         goto out_fail;
10061                 }
10062         }
10063
10064         ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
10065                              new_dentry->d_name.name,
10066                              new_dentry->d_name.len, 0, index);
10067         if (ret) {
10068                 btrfs_abort_transaction(trans, ret);
10069                 goto out_fail;
10070         }
10071
10072         if (old_inode->i_nlink == 1)
10073                 BTRFS_I(old_inode)->dir_index = index;
10074
10075         if (log_pinned) {
10076                 struct dentry *parent = new_dentry->d_parent;
10077
10078                 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
10079                                 parent);
10080                 btrfs_end_log_trans(root);
10081                 log_pinned = false;
10082         }
10083
10084         if (flags & RENAME_WHITEOUT) {
10085                 ret = btrfs_whiteout_for_rename(trans, root, old_dir,
10086                                                 old_dentry);
10087
10088                 if (ret) {
10089                         btrfs_abort_transaction(trans, ret);
10090                         goto out_fail;
10091                 }
10092         }
10093 out_fail:
10094         /*
10095          * If we have pinned the log and an error happened, we unpin tasks
10096          * trying to sync the log and force them to fallback to a transaction
10097          * commit if the log currently contains any of the inodes involved in
10098          * this rename operation (to ensure we do not persist a log with an
10099          * inconsistent state for any of these inodes or leading to any
10100          * inconsistencies when replayed). If the transaction was aborted, the
10101          * abortion reason is propagated to userspace when attempting to commit
10102          * the transaction. If the log does not contain any of these inodes, we
10103          * allow the tasks to sync it.
10104          */
10105         if (ret && log_pinned) {
10106                 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) ||
10107                     btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) ||
10108                     btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) ||
10109                     (new_inode &&
10110                      btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation)))
10111                         btrfs_set_log_full_commit(fs_info, trans);
10112
10113                 btrfs_end_log_trans(root);
10114                 log_pinned = false;
10115         }
10116         btrfs_end_transaction(trans);
10117 out_notrans:
10118         if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
10119                 up_read(&fs_info->subvol_sem);
10120
10121         return ret;
10122 }
10123
10124 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry,
10125                          struct inode *new_dir, struct dentry *new_dentry,
10126                          unsigned int flags)
10127 {
10128         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
10129                 return -EINVAL;
10130
10131         if (flags & RENAME_EXCHANGE)
10132                 return btrfs_rename_exchange(old_dir, old_dentry, new_dir,
10133                                           new_dentry);
10134
10135         return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags);
10136 }
10137
10138 static void btrfs_run_delalloc_work(struct btrfs_work *work)
10139 {
10140         struct btrfs_delalloc_work *delalloc_work;
10141         struct inode *inode;
10142
10143         delalloc_work = container_of(work, struct btrfs_delalloc_work,
10144                                      work);
10145         inode = delalloc_work->inode;
10146         filemap_flush(inode->i_mapping);
10147         if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
10148                                 &BTRFS_I(inode)->runtime_flags))
10149                 filemap_flush(inode->i_mapping);
10150
10151         if (delalloc_work->delay_iput)
10152                 btrfs_add_delayed_iput(inode);
10153         else
10154                 iput(inode);
10155         complete(&delalloc_work->completion);
10156 }
10157
10158 struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode,
10159                                                     int delay_iput)
10160 {
10161         struct btrfs_delalloc_work *work;
10162
10163         work = kmalloc(sizeof(*work), GFP_NOFS);
10164         if (!work)
10165                 return NULL;
10166
10167         init_completion(&work->completion);
10168         INIT_LIST_HEAD(&work->list);
10169         work->inode = inode;
10170         work->delay_iput = delay_iput;
10171         WARN_ON_ONCE(!inode);
10172         btrfs_init_work(&work->work, btrfs_flush_delalloc_helper,
10173                         btrfs_run_delalloc_work, NULL, NULL);
10174
10175         return work;
10176 }
10177
10178 void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work)
10179 {
10180         wait_for_completion(&work->completion);
10181         kfree(work);
10182 }
10183
10184 /*
10185  * some fairly slow code that needs optimization. This walks the list
10186  * of all the inodes with pending delalloc and forces them to disk.
10187  */
10188 static int __start_delalloc_inodes(struct btrfs_root *root, int delay_iput,
10189                                    int nr)
10190 {
10191         struct btrfs_inode *binode;
10192         struct inode *inode;
10193         struct btrfs_delalloc_work *work, *next;
10194         struct list_head works;
10195         struct list_head splice;
10196         int ret = 0;
10197
10198         INIT_LIST_HEAD(&works);
10199         INIT_LIST_HEAD(&splice);
10200
10201         mutex_lock(&root->delalloc_mutex);
10202         spin_lock(&root->delalloc_lock);
10203         list_splice_init(&root->delalloc_inodes, &splice);
10204         while (!list_empty(&splice)) {
10205                 binode = list_entry(splice.next, struct btrfs_inode,
10206                                     delalloc_inodes);
10207
10208                 list_move_tail(&binode->delalloc_inodes,
10209                                &root->delalloc_inodes);
10210                 inode = igrab(&binode->vfs_inode);
10211                 if (!inode) {
10212                         cond_resched_lock(&root->delalloc_lock);
10213                         continue;
10214                 }
10215                 spin_unlock(&root->delalloc_lock);
10216
10217                 work = btrfs_alloc_delalloc_work(inode, delay_iput);
10218                 if (!work) {
10219                         if (delay_iput)
10220                                 btrfs_add_delayed_iput(inode);
10221                         else
10222                                 iput(inode);
10223                         ret = -ENOMEM;
10224                         goto out;
10225                 }
10226                 list_add_tail(&work->list, &works);
10227                 btrfs_queue_work(root->fs_info->flush_workers,
10228                                  &work->work);
10229                 ret++;
10230                 if (nr != -1 && ret >= nr)
10231                         goto out;
10232                 cond_resched();
10233                 spin_lock(&root->delalloc_lock);
10234         }
10235         spin_unlock(&root->delalloc_lock);
10236
10237 out:
10238         list_for_each_entry_safe(work, next, &works, list) {
10239                 list_del_init(&work->list);
10240                 btrfs_wait_and_free_delalloc_work(work);
10241         }
10242
10243         if (!list_empty_careful(&splice)) {
10244                 spin_lock(&root->delalloc_lock);
10245                 list_splice_tail(&splice, &root->delalloc_inodes);
10246                 spin_unlock(&root->delalloc_lock);
10247         }
10248         mutex_unlock(&root->delalloc_mutex);
10249         return ret;
10250 }
10251
10252 int btrfs_start_delalloc_inodes(struct btrfs_root *root, int delay_iput)
10253 {
10254         struct btrfs_fs_info *fs_info = root->fs_info;
10255         int ret;
10256
10257         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
10258                 return -EROFS;
10259
10260         ret = __start_delalloc_inodes(root, delay_iput, -1);
10261         if (ret > 0)
10262                 ret = 0;
10263         /*
10264          * the filemap_flush will queue IO into the worker threads, but
10265          * we have to make sure the IO is actually started and that
10266          * ordered extents get created before we return
10267          */
10268         atomic_inc(&fs_info->async_submit_draining);
10269         while (atomic_read(&fs_info->nr_async_submits) ||
10270                atomic_read(&fs_info->async_delalloc_pages)) {
10271                 wait_event(fs_info->async_submit_wait,
10272                            (atomic_read(&fs_info->nr_async_submits) == 0 &&
10273                             atomic_read(&fs_info->async_delalloc_pages) == 0));
10274         }
10275         atomic_dec(&fs_info->async_submit_draining);
10276         return ret;
10277 }
10278
10279 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int delay_iput,
10280                                int nr)
10281 {
10282         struct btrfs_root *root;
10283         struct list_head splice;
10284         int ret;
10285
10286         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
10287                 return -EROFS;
10288
10289         INIT_LIST_HEAD(&splice);
10290
10291         mutex_lock(&fs_info->delalloc_root_mutex);
10292         spin_lock(&fs_info->delalloc_root_lock);
10293         list_splice_init(&fs_info->delalloc_roots, &splice);
10294         while (!list_empty(&splice) && nr) {
10295                 root = list_first_entry(&splice, struct btrfs_root,
10296                                         delalloc_root);
10297                 root = btrfs_grab_fs_root(root);
10298                 BUG_ON(!root);
10299                 list_move_tail(&root->delalloc_root,
10300                                &fs_info->delalloc_roots);
10301                 spin_unlock(&fs_info->delalloc_root_lock);
10302
10303                 ret = __start_delalloc_inodes(root, delay_iput, nr);
10304                 btrfs_put_fs_root(root);
10305                 if (ret < 0)
10306                         goto out;
10307
10308                 if (nr != -1) {
10309                         nr -= ret;
10310                         WARN_ON(nr < 0);
10311                 }
10312                 spin_lock(&fs_info->delalloc_root_lock);
10313         }
10314         spin_unlock(&fs_info->delalloc_root_lock);
10315
10316         ret = 0;
10317         atomic_inc(&fs_info->async_submit_draining);
10318         while (atomic_read(&fs_info->nr_async_submits) ||
10319               atomic_read(&fs_info->async_delalloc_pages)) {
10320                 wait_event(fs_info->async_submit_wait,
10321                    (atomic_read(&fs_info->nr_async_submits) == 0 &&
10322                     atomic_read(&fs_info->async_delalloc_pages) == 0));
10323         }
10324         atomic_dec(&fs_info->async_submit_draining);
10325 out:
10326         if (!list_empty_careful(&splice)) {
10327                 spin_lock(&fs_info->delalloc_root_lock);
10328                 list_splice_tail(&splice, &fs_info->delalloc_roots);
10329                 spin_unlock(&fs_info->delalloc_root_lock);
10330         }
10331         mutex_unlock(&fs_info->delalloc_root_mutex);
10332         return ret;
10333 }
10334
10335 static int btrfs_symlink(struct inode *dir, struct dentry *dentry,
10336                          const char *symname)
10337 {
10338         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
10339         struct btrfs_trans_handle *trans;
10340         struct btrfs_root *root = BTRFS_I(dir)->root;
10341         struct btrfs_path *path;
10342         struct btrfs_key key;
10343         struct inode *inode = NULL;
10344         int err;
10345         int drop_inode = 0;
10346         u64 objectid;
10347         u64 index = 0;
10348         int name_len;
10349         int datasize;
10350         unsigned long ptr;
10351         struct btrfs_file_extent_item *ei;
10352         struct extent_buffer *leaf;
10353
10354         name_len = strlen(symname);
10355         if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info))
10356                 return -ENAMETOOLONG;
10357
10358         /*
10359          * 2 items for inode item and ref
10360          * 2 items for dir items
10361          * 1 item for updating parent inode item
10362          * 1 item for the inline extent item
10363          * 1 item for xattr if selinux is on
10364          */
10365         trans = btrfs_start_transaction(root, 7);
10366         if (IS_ERR(trans))
10367                 return PTR_ERR(trans);
10368
10369         err = btrfs_find_free_ino(root, &objectid);
10370         if (err)
10371                 goto out_unlock;
10372
10373         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
10374                                 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)),
10375                                 objectid, S_IFLNK|S_IRWXUGO, &index);
10376         if (IS_ERR(inode)) {
10377                 err = PTR_ERR(inode);
10378                 goto out_unlock;
10379         }
10380
10381         /*
10382         * If the active LSM wants to access the inode during
10383         * d_instantiate it needs these. Smack checks to see
10384         * if the filesystem supports xattrs by looking at the
10385         * ops vector.
10386         */
10387         inode->i_fop = &btrfs_file_operations;
10388         inode->i_op = &btrfs_file_inode_operations;
10389         inode->i_mapping->a_ops = &btrfs_aops;
10390         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
10391
10392         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
10393         if (err)
10394                 goto out_unlock_inode;
10395
10396         path = btrfs_alloc_path();
10397         if (!path) {
10398                 err = -ENOMEM;
10399                 goto out_unlock_inode;
10400         }
10401         key.objectid = btrfs_ino(BTRFS_I(inode));
10402         key.offset = 0;
10403         key.type = BTRFS_EXTENT_DATA_KEY;
10404         datasize = btrfs_file_extent_calc_inline_size(name_len);
10405         err = btrfs_insert_empty_item(trans, root, path, &key,
10406                                       datasize);
10407         if (err) {
10408                 btrfs_free_path(path);
10409                 goto out_unlock_inode;
10410         }
10411         leaf = path->nodes[0];
10412         ei = btrfs_item_ptr(leaf, path->slots[0],
10413                             struct btrfs_file_extent_item);
10414         btrfs_set_file_extent_generation(leaf, ei, trans->transid);
10415         btrfs_set_file_extent_type(leaf, ei,
10416                                    BTRFS_FILE_EXTENT_INLINE);
10417         btrfs_set_file_extent_encryption(leaf, ei, 0);
10418         btrfs_set_file_extent_compression(leaf, ei, 0);
10419         btrfs_set_file_extent_other_encoding(leaf, ei, 0);
10420         btrfs_set_file_extent_ram_bytes(leaf, ei, name_len);
10421
10422         ptr = btrfs_file_extent_inline_start(ei);
10423         write_extent_buffer(leaf, symname, ptr, name_len);
10424         btrfs_mark_buffer_dirty(leaf);
10425         btrfs_free_path(path);
10426
10427         inode->i_op = &btrfs_symlink_inode_operations;
10428         inode_nohighmem(inode);
10429         inode->i_mapping->a_ops = &btrfs_symlink_aops;
10430         inode_set_bytes(inode, name_len);
10431         btrfs_i_size_write(BTRFS_I(inode), name_len);
10432         err = btrfs_update_inode(trans, root, inode);
10433         /*
10434          * Last step, add directory indexes for our symlink inode. This is the
10435          * last step to avoid extra cleanup of these indexes if an error happens
10436          * elsewhere above.
10437          */
10438         if (!err)
10439                 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
10440                                 BTRFS_I(inode), 0, index);
10441         if (err) {
10442                 drop_inode = 1;
10443                 goto out_unlock_inode;
10444         }
10445
10446         unlock_new_inode(inode);
10447         d_instantiate(dentry, inode);
10448
10449 out_unlock:
10450         btrfs_end_transaction(trans);
10451         if (drop_inode) {
10452                 inode_dec_link_count(inode);
10453                 iput(inode);
10454         }
10455         btrfs_btree_balance_dirty(fs_info);
10456         return err;
10457
10458 out_unlock_inode:
10459         drop_inode = 1;
10460         unlock_new_inode(inode);
10461         goto out_unlock;
10462 }
10463
10464 static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
10465                                        u64 start, u64 num_bytes, u64 min_size,
10466                                        loff_t actual_len, u64 *alloc_hint,
10467                                        struct btrfs_trans_handle *trans)
10468 {
10469         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
10470         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
10471         struct extent_map *em;
10472         struct btrfs_root *root = BTRFS_I(inode)->root;
10473         struct btrfs_key ins;
10474         u64 cur_offset = start;
10475         u64 i_size;
10476         u64 cur_bytes;
10477         u64 last_alloc = (u64)-1;
10478         int ret = 0;
10479         bool own_trans = true;
10480         u64 end = start + num_bytes - 1;
10481
10482         if (trans)
10483                 own_trans = false;
10484         while (num_bytes > 0) {
10485                 if (own_trans) {
10486                         trans = btrfs_start_transaction(root, 3);
10487                         if (IS_ERR(trans)) {
10488                                 ret = PTR_ERR(trans);
10489                                 break;
10490                         }
10491                 }
10492
10493                 cur_bytes = min_t(u64, num_bytes, SZ_256M);
10494                 cur_bytes = max(cur_bytes, min_size);
10495                 /*
10496                  * If we are severely fragmented we could end up with really
10497                  * small allocations, so if the allocator is returning small
10498                  * chunks lets make its job easier by only searching for those
10499                  * sized chunks.
10500                  */
10501                 cur_bytes = min(cur_bytes, last_alloc);
10502                 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,
10503                                 min_size, 0, *alloc_hint, &ins, 1, 0);
10504                 if (ret) {
10505                         if (own_trans)
10506                                 btrfs_end_transaction(trans);
10507                         break;
10508                 }
10509                 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
10510
10511                 last_alloc = ins.offset;
10512                 ret = insert_reserved_file_extent(trans, inode,
10513                                                   cur_offset, ins.objectid,
10514                                                   ins.offset, ins.offset,
10515                                                   ins.offset, 0, 0, 0,
10516                                                   BTRFS_FILE_EXTENT_PREALLOC);
10517                 if (ret) {
10518                         btrfs_free_reserved_extent(fs_info, ins.objectid,
10519                                                    ins.offset, 0);
10520                         btrfs_abort_transaction(trans, ret);
10521                         if (own_trans)
10522                                 btrfs_end_transaction(trans);
10523                         break;
10524                 }
10525
10526                 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
10527                                         cur_offset + ins.offset -1, 0);
10528
10529                 em = alloc_extent_map();
10530                 if (!em) {
10531                         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
10532                                 &BTRFS_I(inode)->runtime_flags);
10533                         goto next;
10534                 }
10535
10536                 em->start = cur_offset;
10537                 em->orig_start = cur_offset;
10538                 em->len = ins.offset;
10539                 em->block_start = ins.objectid;
10540                 em->block_len = ins.offset;
10541                 em->orig_block_len = ins.offset;
10542                 em->ram_bytes = ins.offset;
10543                 em->bdev = fs_info->fs_devices->latest_bdev;
10544                 set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
10545                 em->generation = trans->transid;
10546
10547                 while (1) {
10548                         write_lock(&em_tree->lock);
10549                         ret = add_extent_mapping(em_tree, em, 1);
10550                         write_unlock(&em_tree->lock);
10551                         if (ret != -EEXIST)
10552                                 break;
10553                         btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
10554                                                 cur_offset + ins.offset - 1,
10555                                                 0);
10556                 }
10557                 free_extent_map(em);
10558 next:
10559                 num_bytes -= ins.offset;
10560                 cur_offset += ins.offset;
10561                 *alloc_hint = ins.objectid + ins.offset;
10562
10563                 inode_inc_iversion(inode);
10564                 inode->i_ctime = current_time(inode);
10565                 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC;
10566                 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
10567                     (actual_len > inode->i_size) &&
10568                     (cur_offset > inode->i_size)) {
10569                         if (cur_offset > actual_len)
10570                                 i_size = actual_len;
10571                         else
10572                                 i_size = cur_offset;
10573                         i_size_write(inode, i_size);
10574                         btrfs_ordered_update_i_size(inode, i_size, NULL);
10575                 }
10576
10577                 ret = btrfs_update_inode(trans, root, inode);
10578
10579                 if (ret) {
10580                         btrfs_abort_transaction(trans, ret);
10581                         if (own_trans)
10582                                 btrfs_end_transaction(trans);
10583                         break;
10584                 }
10585
10586                 if (own_trans)
10587                         btrfs_end_transaction(trans);
10588         }
10589         if (cur_offset < end)
10590                 btrfs_free_reserved_data_space(inode, NULL, cur_offset,
10591                         end - cur_offset + 1);
10592         return ret;
10593 }
10594
10595 int btrfs_prealloc_file_range(struct inode *inode, int mode,
10596                               u64 start, u64 num_bytes, u64 min_size,
10597                               loff_t actual_len, u64 *alloc_hint)
10598 {
10599         return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
10600                                            min_size, actual_len, alloc_hint,
10601                                            NULL);
10602 }
10603
10604 int btrfs_prealloc_file_range_trans(struct inode *inode,
10605                                     struct btrfs_trans_handle *trans, int mode,
10606                                     u64 start, u64 num_bytes, u64 min_size,
10607                                     loff_t actual_len, u64 *alloc_hint)
10608 {
10609         return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
10610                                            min_size, actual_len, alloc_hint, trans);
10611 }
10612
10613 static int btrfs_set_page_dirty(struct page *page)
10614 {
10615         return __set_page_dirty_nobuffers(page);
10616 }
10617
10618 static int btrfs_permission(struct inode *inode, int mask)
10619 {
10620         struct btrfs_root *root = BTRFS_I(inode)->root;
10621         umode_t mode = inode->i_mode;
10622
10623         if (mask & MAY_WRITE &&
10624             (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) {
10625                 if (btrfs_root_readonly(root))
10626                         return -EROFS;
10627                 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)
10628                         return -EACCES;
10629         }
10630         return generic_permission(inode, mask);
10631 }
10632
10633 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
10634 {
10635         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
10636         struct btrfs_trans_handle *trans;
10637         struct btrfs_root *root = BTRFS_I(dir)->root;
10638         struct inode *inode = NULL;
10639         u64 objectid;
10640         u64 index;
10641         int ret = 0;
10642
10643         /*
10644          * 5 units required for adding orphan entry
10645          */
10646         trans = btrfs_start_transaction(root, 5);
10647         if (IS_ERR(trans))
10648                 return PTR_ERR(trans);
10649
10650         ret = btrfs_find_free_ino(root, &objectid);
10651         if (ret)
10652                 goto out;
10653
10654         inode = btrfs_new_inode(trans, root, dir, NULL, 0,
10655                         btrfs_ino(BTRFS_I(dir)), objectid, mode, &index);
10656         if (IS_ERR(inode)) {
10657                 ret = PTR_ERR(inode);
10658                 inode = NULL;
10659                 goto out;
10660         }
10661
10662         inode->i_fop = &btrfs_file_operations;
10663         inode->i_op = &btrfs_file_inode_operations;
10664
10665         inode->i_mapping->a_ops = &btrfs_aops;
10666         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
10667
10668         ret = btrfs_init_inode_security(trans, inode, dir, NULL);
10669         if (ret)
10670                 goto out_inode;
10671
10672         ret = btrfs_update_inode(trans, root, inode);
10673         if (ret)
10674                 goto out_inode;
10675         ret = btrfs_orphan_add(trans, BTRFS_I(inode));
10676         if (ret)
10677                 goto out_inode;
10678
10679         /*
10680          * We set number of links to 0 in btrfs_new_inode(), and here we set
10681          * it to 1 because d_tmpfile() will issue a warning if the count is 0,
10682          * through:
10683          *
10684          *    d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
10685          */
10686         set_nlink(inode, 1);
10687         unlock_new_inode(inode);
10688         d_tmpfile(dentry, inode);
10689         mark_inode_dirty(inode);
10690
10691 out:
10692         btrfs_end_transaction(trans);
10693         if (ret)
10694                 iput(inode);
10695         btrfs_balance_delayed_items(fs_info);
10696         btrfs_btree_balance_dirty(fs_info);
10697         return ret;
10698
10699 out_inode:
10700         unlock_new_inode(inode);
10701         goto out;
10702
10703 }
10704
10705 __attribute__((const))
10706 static int btrfs_readpage_io_failed_hook(struct page *page, int failed_mirror)
10707 {
10708         return -EAGAIN;
10709 }
10710
10711 static struct btrfs_fs_info *iotree_fs_info(void *private_data)
10712 {
10713         struct inode *inode = private_data;
10714         return btrfs_sb(inode->i_sb);
10715 }
10716
10717 static void btrfs_check_extent_io_range(void *private_data, const char *caller,
10718                                         u64 start, u64 end)
10719 {
10720         struct inode *inode = private_data;
10721         u64 isize;
10722
10723         isize = i_size_read(inode);
10724         if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
10725                 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
10726                     "%s: ino %llu isize %llu odd range [%llu,%llu]",
10727                         caller, btrfs_ino(BTRFS_I(inode)), isize, start, end);
10728         }
10729 }
10730
10731 void btrfs_set_range_writeback(void *private_data, u64 start, u64 end)
10732 {
10733         struct inode *inode = private_data;
10734         unsigned long index = start >> PAGE_SHIFT;
10735         unsigned long end_index = end >> PAGE_SHIFT;
10736         struct page *page;
10737
10738         while (index <= end_index) {
10739                 page = find_get_page(inode->i_mapping, index);
10740                 ASSERT(page); /* Pages should be in the extent_io_tree */
10741                 set_page_writeback(page);
10742                 put_page(page);
10743                 index++;
10744         }
10745 }
10746
10747 static const struct inode_operations btrfs_dir_inode_operations = {
10748         .getattr        = btrfs_getattr,
10749         .lookup         = btrfs_lookup,
10750         .create         = btrfs_create,
10751         .unlink         = btrfs_unlink,
10752         .link           = btrfs_link,
10753         .mkdir          = btrfs_mkdir,
10754         .rmdir          = btrfs_rmdir,
10755         .rename         = btrfs_rename2,
10756         .symlink        = btrfs_symlink,
10757         .setattr        = btrfs_setattr,
10758         .mknod          = btrfs_mknod,
10759         .listxattr      = btrfs_listxattr,
10760         .permission     = btrfs_permission,
10761         .get_acl        = btrfs_get_acl,
10762         .set_acl        = btrfs_set_acl,
10763         .update_time    = btrfs_update_time,
10764         .tmpfile        = btrfs_tmpfile,
10765 };
10766 static const struct inode_operations btrfs_dir_ro_inode_operations = {
10767         .lookup         = btrfs_lookup,
10768         .permission     = btrfs_permission,
10769         .update_time    = btrfs_update_time,
10770 };
10771
10772 static const struct file_operations btrfs_dir_file_operations = {
10773         .llseek         = generic_file_llseek,
10774         .read           = generic_read_dir,
10775         .iterate_shared = btrfs_real_readdir,
10776         .unlocked_ioctl = btrfs_ioctl,
10777 #ifdef CONFIG_COMPAT
10778         .compat_ioctl   = btrfs_compat_ioctl,
10779 #endif
10780         .release        = btrfs_release_file,
10781         .fsync          = btrfs_sync_file,
10782 };
10783
10784 static const struct extent_io_ops btrfs_extent_io_ops = {
10785         /* mandatory callbacks */
10786         .submit_bio_hook = btrfs_submit_bio_hook,
10787         .readpage_end_io_hook = btrfs_readpage_end_io_hook,
10788         .merge_bio_hook = btrfs_merge_bio_hook,
10789         .readpage_io_failed_hook = btrfs_readpage_io_failed_hook,
10790         .tree_fs_info = iotree_fs_info,
10791         .set_range_writeback = btrfs_set_range_writeback,
10792
10793         /* optional callbacks */
10794         .fill_delalloc = run_delalloc_range,
10795         .writepage_end_io_hook = btrfs_writepage_end_io_hook,
10796         .writepage_start_hook = btrfs_writepage_start_hook,
10797         .set_bit_hook = btrfs_set_bit_hook,
10798         .clear_bit_hook = btrfs_clear_bit_hook,
10799         .merge_extent_hook = btrfs_merge_extent_hook,
10800         .split_extent_hook = btrfs_split_extent_hook,
10801         .check_extent_io_range = btrfs_check_extent_io_range,
10802 };
10803
10804 /*
10805  * btrfs doesn't support the bmap operation because swapfiles
10806  * use bmap to make a mapping of extents in the file.  They assume
10807  * these extents won't change over the life of the file and they
10808  * use the bmap result to do IO directly to the drive.
10809  *
10810  * the btrfs bmap call would return logical addresses that aren't
10811  * suitable for IO and they also will change frequently as COW
10812  * operations happen.  So, swapfile + btrfs == corruption.
10813  *
10814  * For now we're avoiding this by dropping bmap.
10815  */
10816 static const struct address_space_operations btrfs_aops = {
10817         .readpage       = btrfs_readpage,
10818         .writepage      = btrfs_writepage,
10819         .writepages     = btrfs_writepages,
10820         .readpages      = btrfs_readpages,
10821         .direct_IO      = btrfs_direct_IO,
10822         .invalidatepage = btrfs_invalidatepage,
10823         .releasepage    = btrfs_releasepage,
10824         .set_page_dirty = btrfs_set_page_dirty,
10825         .error_remove_page = generic_error_remove_page,
10826 };
10827
10828 static const struct address_space_operations btrfs_symlink_aops = {
10829         .readpage       = btrfs_readpage,
10830         .writepage      = btrfs_writepage,
10831         .invalidatepage = btrfs_invalidatepage,
10832         .releasepage    = btrfs_releasepage,
10833 };
10834
10835 static const struct inode_operations btrfs_file_inode_operations = {
10836         .getattr        = btrfs_getattr,
10837         .setattr        = btrfs_setattr,
10838         .listxattr      = btrfs_listxattr,
10839         .permission     = btrfs_permission,
10840         .fiemap         = btrfs_fiemap,
10841         .get_acl        = btrfs_get_acl,
10842         .set_acl        = btrfs_set_acl,
10843         .update_time    = btrfs_update_time,
10844 };
10845 static const struct inode_operations btrfs_special_inode_operations = {
10846         .getattr        = btrfs_getattr,
10847         .setattr        = btrfs_setattr,
10848         .permission     = btrfs_permission,
10849         .listxattr      = btrfs_listxattr,
10850         .get_acl        = btrfs_get_acl,
10851         .set_acl        = btrfs_set_acl,
10852         .update_time    = btrfs_update_time,
10853 };
10854 static const struct inode_operations btrfs_symlink_inode_operations = {
10855         .get_link       = page_get_link,
10856         .getattr        = btrfs_getattr,
10857         .setattr        = btrfs_setattr,
10858         .permission     = btrfs_permission,
10859         .listxattr      = btrfs_listxattr,
10860         .update_time    = btrfs_update_time,
10861 };
10862
10863 const struct dentry_operations btrfs_dentry_operations = {
10864         .d_delete       = btrfs_dentry_delete,
10865         .d_release      = btrfs_dentry_release,
10866 };