fs/btrfs/file-item.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/bio.h>
   7 #include <linux/slab.h>
   8 #include <linux/pagemap.h>
   9 #include <linux/highmem.h>
  10 #include <linux/sched/mm.h>
  11 #include <crypto/hash.h>
  12 #include "messages.h"
  13 #include "misc.h"
  14 #include "ctree.h"
  15 #include "disk-io.h"
  16 #include "transaction.h"
  17 #include "bio.h"
  18 #include "print-tree.h"
  19 #include "compression.h"
  20 #include "fs.h"
  21 #include "accessors.h"
  22 #include "file-item.h"
  23 #include "super.h"
  24
  25 #define __MAX_CSUM_ITEMS(r, size) ((unsigned long)(((BTRFS_LEAF_DATA_SIZE(r) - \
  26                                    sizeof(struct btrfs_item) * 2) / \
  27                                   size) - 1))
  28
  29 #define MAX_CSUM_ITEMS(r, size) (min_t(u32, __MAX_CSUM_ITEMS(r, size), \
  30                                        PAGE_SIZE))
  31
  32 /*
  33  * Set inode's size according to filesystem options.
  34  *
  35  * @inode:      inode we want to update the disk_i_size for
  36  * @new_i_size: i_size we want to set to, 0 if we use i_size
  37  *
  38  * With NO_HOLES set this simply sets the disk_is_size to whatever i_size_read()
  39  * returns as it is perfectly fine with a file that has holes without hole file
  40  * extent items.
  41  *
  42  * However without NO_HOLES we need to only return the area that is contiguous
  43  * from the 0 offset of the file.  Otherwise we could end up adjust i_size up
  44  * to an extent that has a gap in between.
  45  *
  46  * Finally new_i_size should only be set in the case of truncate where we're not
  47  * ready to use i_size_read() as the limiter yet.
  48  */
  49 void btrfs_inode_safe_disk_i_size_write(struct btrfs_inode *inode, u64 new_i_size)
  50 {
  51         struct btrfs_fs_info *fs_info = inode->root->fs_info;
  52         u64 start, end, i_size;
  53         int ret;
  54
  55         spin_lock(&inode->lock);
  56         i_size = new_i_size ?: i_size_read(&inode->vfs_inode);
  57         if (btrfs_fs_incompat(fs_info, NO_HOLES)) {
  58                 inode->disk_i_size = i_size;
  59                 goto out_unlock;
  60         }
  61
  62         ret = find_contiguous_extent_bit(&inode->file_extent_tree, 0, &start,
  63                                          &end, EXTENT_DIRTY);
  64         if (!ret && start == 0)
  65                 i_size = min(i_size, end + 1);
  66         else
  67                 i_size = 0;
  68         inode->disk_i_size = i_size;
  69 out_unlock:
  70         spin_unlock(&inode->lock);
  71 }
  72
  73 /*
  74  * Mark range within a file as having a new extent inserted.
  75  *
  76  * @inode: inode being modified
  77  * @start: start file offset of the file extent we've inserted
  78  * @len:   logical length of the file extent item
  79  *
  80  * Call when we are inserting a new file extent where there was none before.
  81  * Does not need to call this in the case where we're replacing an existing file
  82  * extent, however if not sure it's fine to call this multiple times.
  83  *
  84  * The start and len must match the file extent item, so thus must be sectorsize
  85  * aligned.
  86  */
  87 int btrfs_inode_set_file_extent_range(struct btrfs_inode *inode, u64 start,
  88                                       u64 len)
  89 {
  90         if (len == 0)
  91                 return 0;
  92
  93         ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize));
  94
  95         if (btrfs_fs_incompat(inode->root->fs_info, NO_HOLES))
  96                 return 0;
  97         return set_extent_bit(&inode->file_extent_tree, start, start + len - 1,
  98                               EXTENT_DIRTY, NULL);
  99 }
 100
 101 /*
 102  * Mark an inode range as not having a backing extent.
 103  *
 104  * @inode: inode being modified
 105  * @start: start file offset of the file extent we've inserted
 106  * @len:   logical length of the file extent item
 107  *
 108  * Called when we drop a file extent, for example when we truncate.  Doesn't
 109  * need to be called for cases where we're replacing a file extent, like when
 110  * we've COWed a file extent.
 111  *
 112  * The start and len must match the file extent item, so thus must be sectorsize
 113  * aligned.
 114  */
 115 int btrfs_inode_clear_file_extent_range(struct btrfs_inode *inode, u64 start,
 116                                         u64 len)
 117 {
 118         if (len == 0)
 119                 return 0;
 120
 121         ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize) ||
 122                len == (u64)-1);
 123
 124         if (btrfs_fs_incompat(inode->root->fs_info, NO_HOLES))
 125                 return 0;
 126         return clear_extent_bit(&inode->file_extent_tree, start,
 127                                 start + len - 1, EXTENT_DIRTY, NULL);
 128 }
 129
 130 static size_t bytes_to_csum_size(const struct btrfs_fs_info *fs_info, u32 bytes)
 131 {
 132         ASSERT(IS_ALIGNED(bytes, fs_info->sectorsize));
 133
 134         return (bytes >> fs_info->sectorsize_bits) * fs_info->csum_size;
 135 }
 136
 137 static size_t csum_size_to_bytes(const struct btrfs_fs_info *fs_info, u32 csum_size)
 138 {
 139         ASSERT(IS_ALIGNED(csum_size, fs_info->csum_size));
 140
 141         return (csum_size / fs_info->csum_size) << fs_info->sectorsize_bits;
 142 }
 143
 144 static inline u32 max_ordered_sum_bytes(const struct btrfs_fs_info *fs_info)
 145 {
 146         u32 max_csum_size = round_down(PAGE_SIZE - sizeof(struct btrfs_ordered_sum),
 147                                        fs_info->csum_size);
 148
 149         return csum_size_to_bytes(fs_info, max_csum_size);
 150 }
 151
 152 /*
 153  * Calculate the total size needed to allocate for an ordered sum structure
 154  * spanning @bytes in the file.
 155  */
 156 static int btrfs_ordered_sum_size(struct btrfs_fs_info *fs_info, unsigned long bytes)
 157 {
 158         return sizeof(struct btrfs_ordered_sum) + bytes_to_csum_size(fs_info, bytes);
 159 }
 160
 161 int btrfs_insert_hole_extent(struct btrfs_trans_handle *trans,
 162                              struct btrfs_root *root,
 163                              u64 objectid, u64 pos, u64 num_bytes)
 164 {
 165         int ret = 0;
 166         struct btrfs_file_extent_item *item;
 167         struct btrfs_key file_key;
 168         struct btrfs_path *path;
 169         struct extent_buffer *leaf;
 170
 171         path = btrfs_alloc_path();
 172         if (!path)
 173                 return -ENOMEM;
 174         file_key.objectid = objectid;
 175         file_key.offset = pos;
 176         file_key.type = BTRFS_EXTENT_DATA_KEY;
 177
 178         ret = btrfs_insert_empty_item(trans, root, path, &file_key,
 179                                       sizeof(*item));
 180         if (ret < 0)
 181                 goto out;
 182         BUG_ON(ret); /* Can't happen */
 183         leaf = path->nodes[0];
 184         item = btrfs_item_ptr(leaf, path->slots[0],
 185                               struct btrfs_file_extent_item);
 186         btrfs_set_file_extent_disk_bytenr(leaf, item, 0);
 187         btrfs_set_file_extent_disk_num_bytes(leaf, item, 0);
 188         btrfs_set_file_extent_offset(leaf, item, 0);
 189         btrfs_set_file_extent_num_bytes(leaf, item, num_bytes);
 190         btrfs_set_file_extent_ram_bytes(leaf, item, num_bytes);
 191         btrfs_set_file_extent_generation(leaf, item, trans->transid);
 192         btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
 193         btrfs_set_file_extent_compression(leaf, item, 0);
 194         btrfs_set_file_extent_encryption(leaf, item, 0);
 195         btrfs_set_file_extent_other_encoding(leaf, item, 0);
 196
 197         btrfs_mark_buffer_dirty(leaf);
 198 out:
 199         btrfs_free_path(path);
 200         return ret;
 201 }
 202
 203 static struct btrfs_csum_item *
 204 btrfs_lookup_csum(struct btrfs_trans_handle *trans,
 205                   struct btrfs_root *root,
 206                   struct btrfs_path *path,
 207                   u64 bytenr, int cow)
 208 {
 209         struct btrfs_fs_info *fs_info = root->fs_info;
 210         int ret;
 211         struct btrfs_key file_key;
 212         struct btrfs_key found_key;
 213         struct btrfs_csum_item *item;
 214         struct extent_buffer *leaf;
 215         u64 csum_offset = 0;
 216         const u32 csum_size = fs_info->csum_size;
 217         int csums_in_item;
 218
 219         file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
 220         file_key.offset = bytenr;
 221         file_key.type = BTRFS_EXTENT_CSUM_KEY;
 222         ret = btrfs_search_slot(trans, root, &file_key, path, 0, cow);
 223         if (ret < 0)
 224                 goto fail;
 225         leaf = path->nodes[0];
 226         if (ret > 0) {
 227                 ret = 1;
 228                 if (path->slots[0] == 0)
 229                         goto fail;
 230                 path->slots[0]--;
 231                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
 232                 if (found_key.type != BTRFS_EXTENT_CSUM_KEY)
 233                         goto fail;
 234
 235                 csum_offset = (bytenr - found_key.offset) >>
 236                                 fs_info->sectorsize_bits;
 237                 csums_in_item = btrfs_item_size(leaf, path->slots[0]);
 238                 csums_in_item /= csum_size;
 239
 240                 if (csum_offset == csums_in_item) {
 241                         ret = -EFBIG;
 242                         goto fail;
 243                 } else if (csum_offset > csums_in_item) {
 244                         goto fail;
 245                 }
 246         }
 247         item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item);
 248         item = (struct btrfs_csum_item *)((unsigned char *)item +
 249                                           csum_offset * csum_size);
 250         return item;
 251 fail:
 252         if (ret > 0)
 253                 ret = -ENOENT;
 254         return ERR_PTR(ret);
 255 }
 256
 257 int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans,
 258                              struct btrfs_root *root,
 259                              struct btrfs_path *path, u64 objectid,
 260                              u64 offset, int mod)
 261 {
 262         struct btrfs_key file_key;
 263         int ins_len = mod < 0 ? -1 : 0;
 264         int cow = mod != 0;
 265
 266         file_key.objectid = objectid;
 267         file_key.offset = offset;
 268         file_key.type = BTRFS_EXTENT_DATA_KEY;
 269
 270         return btrfs_search_slot(trans, root, &file_key, path, ins_len, cow);
 271 }
 272
 273 /*
 274  * Find checksums for logical bytenr range [disk_bytenr, disk_bytenr + len) and
 275  * store the result to @dst.
 276  *
 277  * Return >0 for the number of sectors we found.
 278  * Return 0 for the range [disk_bytenr, disk_bytenr + sectorsize) has no csum
 279  * for it. Caller may want to try next sector until one range is hit.
 280  * Return <0 for fatal error.
 281  */
 282 static int search_csum_tree(struct btrfs_fs_info *fs_info,
 283                             struct btrfs_path *path, u64 disk_bytenr,
 284                             u64 len, u8 *dst)
 285 {
 286         struct btrfs_root *csum_root;
 287         struct btrfs_csum_item *item = NULL;
 288         struct btrfs_key key;
 289         const u32 sectorsize = fs_info->sectorsize;
 290         const u32 csum_size = fs_info->csum_size;
 291         u32 itemsize;
 292         int ret;
 293         u64 csum_start;
 294         u64 csum_len;
 295
 296         ASSERT(IS_ALIGNED(disk_bytenr, sectorsize) &&
 297                IS_ALIGNED(len, sectorsize));
 298
 299         /* Check if the current csum item covers disk_bytenr */
 300         if (path->nodes[0]) {
 301                 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
 302                                       struct btrfs_csum_item);
 303                 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
 304                 itemsize = btrfs_item_size(path->nodes[0], path->slots[0]);
 305
 306                 csum_start = key.offset;
 307                 csum_len = (itemsize / csum_size) * sectorsize;
 308
 309                 if (in_range(disk_bytenr, csum_start, csum_len))
 310                         goto found;
 311         }
 312
 313         /* Current item doesn't contain the desired range, search again */
 314         btrfs_release_path(path);
 315         csum_root = btrfs_csum_root(fs_info, disk_bytenr);
 316         item = btrfs_lookup_csum(NULL, csum_root, path, disk_bytenr, 0);
 317         if (IS_ERR(item)) {
 318                 ret = PTR_ERR(item);
 319                 goto out;
 320         }
 321         btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
 322         itemsize = btrfs_item_size(path->nodes[0], path->slots[0]);
 323
 324         csum_start = key.offset;
 325         csum_len = (itemsize / csum_size) * sectorsize;
 326         ASSERT(in_range(disk_bytenr, csum_start, csum_len));
 327
 328 found:
 329         ret = (min(csum_start + csum_len, disk_bytenr + len) -
 330                    disk_bytenr) >> fs_info->sectorsize_bits;
 331         read_extent_buffer(path->nodes[0], dst, (unsigned long)item,
 332                         ret * csum_size);
 333 out:
 334         if (ret == -ENOENT || ret == -EFBIG)
 335                 ret = 0;
 336         return ret;
 337 }
 338
 339 /*
 340  * Lookup the checksum for the read bio in csum tree.
 341  *
 342  * Return: BLK_STS_RESOURCE if allocating memory fails, BLK_STS_OK otherwise.
 343  */
 344 blk_status_t btrfs_lookup_bio_sums(struct btrfs_bio *bbio)
 345 {
 346         struct btrfs_inode *inode = bbio->inode;
 347         struct btrfs_fs_info *fs_info = inode->root->fs_info;
 348         struct bio *bio = &bbio->bio;
 349         struct btrfs_path *path;
 350         const u32 sectorsize = fs_info->sectorsize;
 351         const u32 csum_size = fs_info->csum_size;
 352         u32 orig_len = bio->bi_iter.bi_size;
 353         u64 orig_disk_bytenr = bio->bi_iter.bi_sector << SECTOR_SHIFT;
 354         const unsigned int nblocks = orig_len >> fs_info->sectorsize_bits;
 355         blk_status_t ret = BLK_STS_OK;
 356         u32 bio_offset = 0;
 357
 358         if ((inode->flags & BTRFS_INODE_NODATASUM) ||
 359             test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state))
 360                 return BLK_STS_OK;
 361
 362         /*
 363          * This function is only called for read bio.
 364          *
 365          * This means two things:
 366          * - All our csums should only be in csum tree
 367          *   No ordered extents csums, as ordered extents are only for write
 368          *   path.
 369          * - No need to bother any other info from bvec
 370          *   Since we're looking up csums, the only important info is the
 371          *   disk_bytenr and the length, which can be extracted from bi_iter
 372          *   directly.
 373          */
 374         ASSERT(bio_op(bio) == REQ_OP_READ);
 375         path = btrfs_alloc_path();
 376         if (!path)
 377                 return BLK_STS_RESOURCE;
 378
 379         if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) {
 380                 bbio->csum = kmalloc_array(nblocks, csum_size, GFP_NOFS);
 381                 if (!bbio->csum) {
 382                         btrfs_free_path(path);
 383                         return BLK_STS_RESOURCE;
 384                 }
 385         } else {
 386                 bbio->csum = bbio->csum_inline;
 387         }
 388
 389         /*
 390          * If requested number of sectors is larger than one leaf can contain,
 391          * kick the readahead for csum tree.
 392          */
 393         if (nblocks > fs_info->csums_per_leaf)
 394                 path->reada = READA_FORWARD;
 395
 396         /*
 397          * the free space stuff is only read when it hasn't been
 398          * updated in the current transaction.  So, we can safely
 399          * read from the commit root and sidestep a nasty deadlock
 400          * between reading the free space cache and updating the csum tree.
 401          */
 402         if (btrfs_is_free_space_inode(inode)) {
 403                 path->search_commit_root = 1;
 404                 path->skip_locking = 1;
 405         }
 406
 407         while (bio_offset < orig_len) {
 408                 int count;
 409                 u64 cur_disk_bytenr = orig_disk_bytenr + bio_offset;
 410                 u8 *csum_dst = bbio->csum +
 411                         (bio_offset >> fs_info->sectorsize_bits) * csum_size;
 412
 413                 count = search_csum_tree(fs_info, path, cur_disk_bytenr,
 414                                          orig_len - bio_offset, csum_dst);
 415                 if (count < 0) {
 416                         ret = errno_to_blk_status(count);
 417                         if (bbio->csum != bbio->csum_inline)
 418                                 kfree(bbio->csum);
 419                         bbio->csum = NULL;
 420                         break;
 421                 }
 422
 423                 /*
 424                  * We didn't find a csum for this range.  We need to make sure
 425                  * we complain loudly about this, because we are not NODATASUM.
 426                  *
 427                  * However for the DATA_RELOC inode we could potentially be
 428                  * relocating data extents for a NODATASUM inode, so the inode
 429                  * itself won't be marked with NODATASUM, but the extent we're
 430                  * copying is in fact NODATASUM.  If we don't find a csum we
 431                  * assume this is the case.
 432                  */
 433                 if (count == 0) {
 434                         memset(csum_dst, 0, csum_size);
 435                         count = 1;
 436
 437                         if (inode->root->root_key.objectid ==
 438                             BTRFS_DATA_RELOC_TREE_OBJECTID) {
 439                                 u64 file_offset = bbio->file_offset + bio_offset;
 440
 441                                 set_extent_bit(&inode->io_tree, file_offset,
 442                                                file_offset + sectorsize - 1,
 443                                                EXTENT_NODATASUM, NULL);
 444                         } else {
 445                                 btrfs_warn_rl(fs_info,
 446                         "csum hole found for disk bytenr range [%llu, %llu)",
 447                                 cur_disk_bytenr, cur_disk_bytenr + sectorsize);
 448                         }
 449                 }
 450                 bio_offset += count * sectorsize;
 451         }
 452
 453         btrfs_free_path(path);
 454         return ret;
 455 }
 456
 457 int btrfs_lookup_csums_list(struct btrfs_root *root, u64 start, u64 end,
 458                             struct list_head *list, int search_commit,
 459                             bool nowait)
 460 {
 461         struct btrfs_fs_info *fs_info = root->fs_info;
 462         struct btrfs_key key;
 463         struct btrfs_path *path;
 464         struct extent_buffer *leaf;
 465         struct btrfs_ordered_sum *sums;
 466         struct btrfs_csum_item *item;
 467         LIST_HEAD(tmplist);
 468         int ret;
 469
 470         ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
 471                IS_ALIGNED(end + 1, fs_info->sectorsize));
 472
 473         path = btrfs_alloc_path();
 474         if (!path)
 475                 return -ENOMEM;
 476
 477         path->nowait = nowait;
 478         if (search_commit) {
 479                 path->skip_locking = 1;
 480                 path->reada = READA_FORWARD;
 481                 path->search_commit_root = 1;
 482         }
 483
 484         key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
 485         key.offset = start;
 486         key.type = BTRFS_EXTENT_CSUM_KEY;
 487
 488         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
 489         if (ret < 0)
 490                 goto fail;
 491         if (ret > 0 && path->slots[0] > 0) {
 492                 leaf = path->nodes[0];
 493                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
 494
 495                 /*
 496                  * There are two cases we can hit here for the previous csum
 497                  * item:
 498                  *
 499                  *              |<- search range ->|
 500                  *      |<- csum item ->|
 501                  *
 502                  * Or
 503                  *                              |<- search range ->|
 504                  *      |<- csum item ->|
 505                  *
 506                  * Check if the previous csum item covers the leading part of
 507                  * the search range.  If so we have to start from previous csum
 508                  * item.
 509                  */
 510                 if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID &&
 511                     key.type == BTRFS_EXTENT_CSUM_KEY) {
 512                         if (bytes_to_csum_size(fs_info, start - key.offset) <
 513                             btrfs_item_size(leaf, path->slots[0] - 1))
 514                                 path->slots[0]--;
 515                 }
 516         }
 517
 518         while (start <= end) {
 519                 u64 csum_end;
 520
 521                 leaf = path->nodes[0];
 522                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
 523                         ret = btrfs_next_leaf(root, path);
 524                         if (ret < 0)
 525                                 goto fail;
 526                         if (ret > 0)
 527                                 break;
 528                         leaf = path->nodes[0];
 529                 }
 530
 531                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 532                 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
 533                     key.type != BTRFS_EXTENT_CSUM_KEY ||
 534                     key.offset > end)
 535                         break;
 536
 537                 if (key.offset > start)
 538                         start = key.offset;
 539
 540                 csum_end = key.offset + csum_size_to_bytes(fs_info,
 541                                         btrfs_item_size(leaf, path->slots[0]));
 542                 if (csum_end <= start) {
 543                         path->slots[0]++;
 544                         continue;
 545                 }
 546
 547                 csum_end = min(csum_end, end + 1);
 548                 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
 549                                       struct btrfs_csum_item);
 550                 while (start < csum_end) {
 551                         unsigned long offset;
 552                         size_t size;
 553
 554                         size = min_t(size_t, csum_end - start,
 555                                      max_ordered_sum_bytes(fs_info));
 556                         sums = kzalloc(btrfs_ordered_sum_size(fs_info, size),
 557                                        GFP_NOFS);
 558                         if (!sums) {
 559                                 ret = -ENOMEM;
 560                                 goto fail;
 561                         }
 562
 563                         sums->logical = start;
 564                         sums->len = size;
 565
 566                         offset = bytes_to_csum_size(fs_info, start - key.offset);
 567
 568                         read_extent_buffer(path->nodes[0],
 569                                            sums->sums,
 570                                            ((unsigned long)item) + offset,
 571                                            bytes_to_csum_size(fs_info, size));
 572
 573                         start += size;
 574                         list_add_tail(&sums->list, &tmplist);
 575                 }
 576                 path->slots[0]++;
 577         }
 578         ret = 0;
 579 fail:
 580         while (ret < 0 && !list_empty(&tmplist)) {
 581                 sums = list_entry(tmplist.next, struct btrfs_ordered_sum, list);
 582                 list_del(&sums->list);
 583                 kfree(sums);
 584         }
 585         list_splice_tail(&tmplist, list);
 586
 587         btrfs_free_path(path);
 588         return ret;
 589 }
 590
 591 /*
 592  * Do the same work as btrfs_lookup_csums_list(), the difference is in how
 593  * we return the result.
 594  *
 595  * This version will set the corresponding bits in @csum_bitmap to represent
 596  * that there is a csum found.
 597  * Each bit represents a sector. Thus caller should ensure @csum_buf passed
 598  * in is large enough to contain all csums.
 599  */
 600 int btrfs_lookup_csums_bitmap(struct btrfs_root *root, u64 start, u64 end,
 601                               u8 *csum_buf, unsigned long *csum_bitmap,
 602                               bool search_commit)
 603 {
 604         struct btrfs_fs_info *fs_info = root->fs_info;
 605         struct btrfs_key key;
 606         struct btrfs_path *path;
 607         struct extent_buffer *leaf;
 608         struct btrfs_csum_item *item;
 609         const u64 orig_start = start;
 610         int ret;
 611
 612         ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
 613                IS_ALIGNED(end + 1, fs_info->sectorsize));
 614
 615         path = btrfs_alloc_path();
 616         if (!path)
 617                 return -ENOMEM;
 618
 619         if (search_commit) {
 620                 path->skip_locking = 1;
 621                 path->reada = READA_FORWARD;
 622                 path->search_commit_root = 1;
 623         }
 624
 625         key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
 626         key.type = BTRFS_EXTENT_CSUM_KEY;
 627         key.offset = start;
 628
 629         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
 630         if (ret < 0)
 631                 goto fail;
 632         if (ret > 0 && path->slots[0] > 0) {
 633                 leaf = path->nodes[0];
 634                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
 635
 636                 /*
 637                  * There are two cases we can hit here for the previous csum
 638                  * item:
 639                  *
 640                  *              |<- search range ->|
 641                  *      |<- csum item ->|
 642                  *
 643                  * Or
 644                  *                              |<- search range ->|
 645                  *      |<- csum item ->|
 646                  *
 647                  * Check if the previous csum item covers the leading part of
 648                  * the search range.  If so we have to start from previous csum
 649                  * item.
 650                  */
 651                 if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID &&
 652                     key.type == BTRFS_EXTENT_CSUM_KEY) {
 653                         if (bytes_to_csum_size(fs_info, start - key.offset) <
 654                             btrfs_item_size(leaf, path->slots[0] - 1))
 655                                 path->slots[0]--;
 656                 }
 657         }
 658
 659         while (start <= end) {
 660                 u64 csum_end;
 661
 662                 leaf = path->nodes[0];
 663                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
 664                         ret = btrfs_next_leaf(root, path);
 665                         if (ret < 0)
 666                                 goto fail;
 667                         if (ret > 0)
 668                                 break;
 669                         leaf = path->nodes[0];
 670                 }
 671
 672                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 673                 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
 674                     key.type != BTRFS_EXTENT_CSUM_KEY ||
 675                     key.offset > end)
 676                         break;
 677
 678                 if (key.offset > start)
 679                         start = key.offset;
 680
 681                 csum_end = key.offset + csum_size_to_bytes(fs_info,
 682                                         btrfs_item_size(leaf, path->slots[0]));
 683                 if (csum_end <= start) {
 684                         path->slots[0]++;
 685                         continue;
 686                 }
 687
 688                 csum_end = min(csum_end, end + 1);
 689                 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
 690                                       struct btrfs_csum_item);
 691                 while (start < csum_end) {
 692                         unsigned long offset;
 693                         size_t size;
 694                         u8 *csum_dest = csum_buf + bytes_to_csum_size(fs_info,
 695                                                 start - orig_start);
 696
 697                         size = min_t(size_t, csum_end - start, end + 1 - start);
 698
 699                         offset = bytes_to_csum_size(fs_info, start - key.offset);
 700
 701                         read_extent_buffer(path->nodes[0], csum_dest,
 702                                            ((unsigned long)item) + offset,
 703                                            bytes_to_csum_size(fs_info, size));
 704
 705                         bitmap_set(csum_bitmap,
 706                                 (start - orig_start) >> fs_info->sectorsize_bits,
 707                                 size >> fs_info->sectorsize_bits);
 708
 709                         start += size;
 710                 }
 711                 path->slots[0]++;
 712         }
 713         ret = 0;
 714 fail:
 715         btrfs_free_path(path);
 716         return ret;
 717 }
 718
 719 /*
 720  * Calculate checksums of the data contained inside a bio.
 721  */
 722 blk_status_t btrfs_csum_one_bio(struct btrfs_bio *bbio)
 723 {
 724         struct btrfs_ordered_extent *ordered = bbio->ordered;
 725         struct btrfs_inode *inode = bbio->inode;
 726         struct btrfs_fs_info *fs_info = inode->root->fs_info;
 727         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
 728         struct bio *bio = &bbio->bio;
 729         struct btrfs_ordered_sum *sums;
 730         char *data;
 731         struct bvec_iter iter;
 732         struct bio_vec bvec;
 733         int index;
 734         unsigned int blockcount;
 735         int i;
 736         unsigned nofs_flag;
 737
 738         nofs_flag = memalloc_nofs_save();
 739         sums = kvzalloc(btrfs_ordered_sum_size(fs_info, bio->bi_iter.bi_size),
 740                        GFP_KERNEL);
 741         memalloc_nofs_restore(nofs_flag);
 742
 743         if (!sums)
 744                 return BLK_STS_RESOURCE;
 745
 746         sums->len = bio->bi_iter.bi_size;
 747         INIT_LIST_HEAD(&sums->list);
 748
 749         sums->logical = bio->bi_iter.bi_sector << SECTOR_SHIFT;
 750         index = 0;
 751
 752         shash->tfm = fs_info->csum_shash;
 753
 754         bio_for_each_segment(bvec, bio, iter) {
 755                 blockcount = BTRFS_BYTES_TO_BLKS(fs_info,
 756                                                  bvec.bv_len + fs_info->sectorsize
 757                                                  - 1);
 758
 759                 for (i = 0; i < blockcount; i++) {
 760                         data = bvec_kmap_local(&bvec);
 761                         crypto_shash_digest(shash,
 762                                             data + (i * fs_info->sectorsize),
 763                                             fs_info->sectorsize,
 764                                             sums->sums + index);
 765                         kunmap_local(data);
 766                         index += fs_info->csum_size;
 767                 }
 768
 769         }
 770
 771         bbio->sums = sums;
 772         btrfs_add_ordered_sum(ordered, sums);
 773         return 0;
 774 }
 775
 776 /*
 777  * Nodatasum I/O on zoned file systems still requires an btrfs_ordered_sum to
 778  * record the updated logical address on Zone Append completion.
 779  * Allocate just the structure with an empty sums array here for that case.
 780  */
 781 blk_status_t btrfs_alloc_dummy_sum(struct btrfs_bio *bbio)
 782 {
 783         bbio->sums = kmalloc(sizeof(*bbio->sums), GFP_NOFS);
 784         if (!bbio->sums)
 785                 return BLK_STS_RESOURCE;
 786         bbio->sums->len = bbio->bio.bi_iter.bi_size;
 787         bbio->sums->logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT;
 788         btrfs_add_ordered_sum(bbio->ordered, bbio->sums);
 789         return 0;
 790 }
 791
 792 /*
 793  * Remove one checksum overlapping a range.
 794  *
 795  * This expects the key to describe the csum pointed to by the path, and it
 796  * expects the csum to overlap the range [bytenr, len]
 797  *
 798  * The csum should not be entirely contained in the range and the range should
 799  * not be entirely contained in the csum.
 800  *
 801  * This calls btrfs_truncate_item with the correct args based on the overlap,
 802  * and fixes up the key as required.
 803  */
 804 static noinline void truncate_one_csum(struct btrfs_fs_info *fs_info,
 805                                        struct btrfs_path *path,
 806                                        struct btrfs_key *key,
 807                                        u64 bytenr, u64 len)
 808 {
 809         struct extent_buffer *leaf;
 810         const u32 csum_size = fs_info->csum_size;
 811         u64 csum_end;
 812         u64 end_byte = bytenr + len;
 813         u32 blocksize_bits = fs_info->sectorsize_bits;
 814
 815         leaf = path->nodes[0];
 816         csum_end = btrfs_item_size(leaf, path->slots[0]) / csum_size;
 817         csum_end <<= blocksize_bits;
 818         csum_end += key->offset;
 819
 820         if (key->offset < bytenr && csum_end <= end_byte) {
 821                 /*
 822                  *         [ bytenr - len ]
 823                  *         [   ]
 824                  *   [csum     ]
 825                  *   A simple truncate off the end of the item
 826                  */
 827                 u32 new_size = (bytenr - key->offset) >> blocksize_bits;
 828                 new_size *= csum_size;
 829                 btrfs_truncate_item(path, new_size, 1);
 830         } else if (key->offset >= bytenr && csum_end > end_byte &&
 831                    end_byte > key->offset) {
 832                 /*
 833                  *         [ bytenr - len ]
 834                  *                 [ ]
 835                  *                 [csum     ]
 836                  * we need to truncate from the beginning of the csum
 837                  */
 838                 u32 new_size = (csum_end - end_byte) >> blocksize_bits;
 839                 new_size *= csum_size;
 840
 841                 btrfs_truncate_item(path, new_size, 0);
 842
 843                 key->offset = end_byte;
 844                 btrfs_set_item_key_safe(fs_info, path, key);
 845         } else {
 846                 BUG();
 847         }
 848 }
 849
 850 /*
 851  * Delete the csum items from the csum tree for a given range of bytes.
 852  */
 853 int btrfs_del_csums(struct btrfs_trans_handle *trans,
 854                     struct btrfs_root *root, u64 bytenr, u64 len)
 855 {
 856         struct btrfs_fs_info *fs_info = trans->fs_info;
 857         struct btrfs_path *path;
 858         struct btrfs_key key;
 859         u64 end_byte = bytenr + len;
 860         u64 csum_end;
 861         struct extent_buffer *leaf;
 862         int ret = 0;
 863         const u32 csum_size = fs_info->csum_size;
 864         u32 blocksize_bits = fs_info->sectorsize_bits;
 865
 866         ASSERT(root->root_key.objectid == BTRFS_CSUM_TREE_OBJECTID ||
 867                root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID);
 868
 869         path = btrfs_alloc_path();
 870         if (!path)
 871                 return -ENOMEM;
 872
 873         while (1) {
 874                 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
 875                 key.offset = end_byte - 1;
 876                 key.type = BTRFS_EXTENT_CSUM_KEY;
 877
 878                 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
 879                 if (ret > 0) {
 880                         ret = 0;
 881                         if (path->slots[0] == 0)
 882                                 break;
 883                         path->slots[0]--;
 884                 } else if (ret < 0) {
 885                         break;
 886                 }
 887
 888                 leaf = path->nodes[0];
 889                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 890
 891                 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
 892                     key.type != BTRFS_EXTENT_CSUM_KEY) {
 893                         break;
 894                 }
 895
 896                 if (key.offset >= end_byte)
 897                         break;
 898
 899                 csum_end = btrfs_item_size(leaf, path->slots[0]) / csum_size;
 900                 csum_end <<= blocksize_bits;
 901                 csum_end += key.offset;
 902
 903                 /* this csum ends before we start, we're done */
 904                 if (csum_end <= bytenr)
 905                         break;
 906
 907                 /* delete the entire item, it is inside our range */
 908                 if (key.offset >= bytenr && csum_end <= end_byte) {
 909                         int del_nr = 1;
 910
 911                         /*
 912                          * Check how many csum items preceding this one in this
 913                          * leaf correspond to our range and then delete them all
 914                          * at once.
 915                          */
 916                         if (key.offset > bytenr && path->slots[0] > 0) {
 917                                 int slot = path->slots[0] - 1;
 918
 919                                 while (slot >= 0) {
 920                                         struct btrfs_key pk;
 921
 922                                         btrfs_item_key_to_cpu(leaf, &pk, slot);
 923                                         if (pk.offset < bytenr ||
 924                                             pk.type != BTRFS_EXTENT_CSUM_KEY ||
 925                                             pk.objectid !=
 926                                             BTRFS_EXTENT_CSUM_OBJECTID)
 927                                                 break;
 928                                         path->slots[0] = slot;
 929                                         del_nr++;
 930                                         key.offset = pk.offset;
 931                                         slot--;
 932                                 }
 933                         }
 934                         ret = btrfs_del_items(trans, root, path,
 935                                               path->slots[0], del_nr);
 936                         if (ret)
 937                                 break;
 938                         if (key.offset == bytenr)
 939                                 break;
 940                 } else if (key.offset < bytenr && csum_end > end_byte) {
 941                         unsigned long offset;
 942                         unsigned long shift_len;
 943                         unsigned long item_offset;
 944                         /*
 945                          *        [ bytenr - len ]
 946                          *     [csum                ]
 947                          *
 948                          * Our bytes are in the middle of the csum,
 949                          * we need to split this item and insert a new one.
 950                          *
 951                          * But we can't drop the path because the
 952                          * csum could change, get removed, extended etc.
 953                          *
 954                          * The trick here is the max size of a csum item leaves
 955                          * enough room in the tree block for a single
 956                          * item header.  So, we split the item in place,
 957                          * adding a new header pointing to the existing
 958                          * bytes.  Then we loop around again and we have
 959                          * a nicely formed csum item that we can neatly
 960                          * truncate.
 961                          */
 962                         offset = (bytenr - key.offset) >> blocksize_bits;
 963                         offset *= csum_size;
 964
 965                         shift_len = (len >> blocksize_bits) * csum_size;
 966
 967                         item_offset = btrfs_item_ptr_offset(leaf,
 968                                                             path->slots[0]);
 969
 970                         memzero_extent_buffer(leaf, item_offset + offset,
 971                                              shift_len);
 972                         key.offset = bytenr;
 973
 974                         /*
 975                          * btrfs_split_item returns -EAGAIN when the
 976                          * item changed size or key
 977                          */
 978                         ret = btrfs_split_item(trans, root, path, &key, offset);
 979                         if (ret && ret != -EAGAIN) {
 980                                 btrfs_abort_transaction(trans, ret);
 981                                 break;
 982                         }
 983                         ret = 0;
 984
 985                         key.offset = end_byte - 1;
 986                 } else {
 987                         truncate_one_csum(fs_info, path, &key, bytenr, len);
 988                         if (key.offset < bytenr)
 989                                 break;
 990                 }
 991                 btrfs_release_path(path);
 992         }
 993         btrfs_free_path(path);
 994         return ret;
 995 }
 996
 997 static int find_next_csum_offset(struct btrfs_root *root,
 998                                  struct btrfs_path *path,
 999                                  u64 *next_offset)
1000 {
1001         const u32 nritems = btrfs_header_nritems(path->nodes[0]);
1002         struct btrfs_key found_key;
1003         int slot = path->slots[0] + 1;
1004         int ret;
1005
1006         if (nritems == 0 || slot >= nritems) {
1007                 ret = btrfs_next_leaf(root, path);
1008                 if (ret < 0) {
1009                         return ret;
1010                 } else if (ret > 0) {
1011                         *next_offset = (u64)-1;
1012                         return 0;
1013                 }
1014                 slot = path->slots[0];
1015         }
1016
1017         btrfs_item_key_to_cpu(path->nodes[0], &found_key, slot);
1018
1019         if (found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
1020             found_key.type != BTRFS_EXTENT_CSUM_KEY)
1021                 *next_offset = (u64)-1;
1022         else
1023                 *next_offset = found_key.offset;
1024
1025         return 0;
1026 }
1027
1028 int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans,
1029                            struct btrfs_root *root,
1030                            struct btrfs_ordered_sum *sums)
1031 {
1032         struct btrfs_fs_info *fs_info = root->fs_info;
1033         struct btrfs_key file_key;
1034         struct btrfs_key found_key;
1035         struct btrfs_path *path;
1036         struct btrfs_csum_item *item;
1037         struct btrfs_csum_item *item_end;
1038         struct extent_buffer *leaf = NULL;
1039         u64 next_offset;
1040         u64 total_bytes = 0;
1041         u64 csum_offset;
1042         u64 bytenr;
1043         u32 ins_size;
1044         int index = 0;
1045         int found_next;
1046         int ret;
1047         const u32 csum_size = fs_info->csum_size;
1048
1049         path = btrfs_alloc_path();
1050         if (!path)
1051                 return -ENOMEM;
1052 again:
1053         next_offset = (u64)-1;
1054         found_next = 0;
1055         bytenr = sums->logical + total_bytes;
1056         file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
1057         file_key.offset = bytenr;
1058         file_key.type = BTRFS_EXTENT_CSUM_KEY;
1059
1060         item = btrfs_lookup_csum(trans, root, path, bytenr, 1);
1061         if (!IS_ERR(item)) {
1062                 ret = 0;
1063                 leaf = path->nodes[0];
1064                 item_end = btrfs_item_ptr(leaf, path->slots[0],
1065                                           struct btrfs_csum_item);
1066                 item_end = (struct btrfs_csum_item *)((char *)item_end +
1067                            btrfs_item_size(leaf, path->slots[0]));
1068                 goto found;
1069         }
1070         ret = PTR_ERR(item);
1071         if (ret != -EFBIG && ret != -ENOENT)
1072                 goto out;
1073
1074         if (ret == -EFBIG) {
1075                 u32 item_size;
1076                 /* we found one, but it isn't big enough yet */
1077                 leaf = path->nodes[0];
1078                 item_size = btrfs_item_size(leaf, path->slots[0]);
1079                 if ((item_size / csum_size) >=
1080                     MAX_CSUM_ITEMS(fs_info, csum_size)) {
1081                         /* already at max size, make a new one */
1082                         goto insert;
1083                 }
1084         } else {
1085                 /* We didn't find a csum item, insert one. */
1086                 ret = find_next_csum_offset(root, path, &next_offset);
1087                 if (ret < 0)
1088                         goto out;
1089                 found_next = 1;
1090                 goto insert;
1091         }
1092
1093         /*
1094          * At this point, we know the tree has a checksum item that ends at an
1095          * offset matching the start of the checksum range we want to insert.
1096          * We try to extend that item as much as possible and then add as many
1097          * checksums to it as they fit.
1098          *
1099          * First check if the leaf has enough free space for at least one
1100          * checksum. If it has go directly to the item extension code, otherwise
1101          * release the path and do a search for insertion before the extension.
1102          */
1103         if (btrfs_leaf_free_space(leaf) >= csum_size) {
1104                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1105                 csum_offset = (bytenr - found_key.offset) >>
1106                         fs_info->sectorsize_bits;
1107                 goto extend_csum;
1108         }
1109
1110         btrfs_release_path(path);
1111         path->search_for_extension = 1;
1112         ret = btrfs_search_slot(trans, root, &file_key, path,
1113                                 csum_size, 1);
1114         path->search_for_extension = 0;
1115         if (ret < 0)
1116                 goto out;
1117
1118         if (ret > 0) {
1119                 if (path->slots[0] == 0)
1120                         goto insert;
1121                 path->slots[0]--;
1122         }
1123
1124         leaf = path->nodes[0];
1125         btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1126         csum_offset = (bytenr - found_key.offset) >> fs_info->sectorsize_bits;
1127
1128         if (found_key.type != BTRFS_EXTENT_CSUM_KEY ||
1129             found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
1130             csum_offset >= MAX_CSUM_ITEMS(fs_info, csum_size)) {
1131                 goto insert;
1132         }
1133
1134 extend_csum:
1135         if (csum_offset == btrfs_item_size(leaf, path->slots[0]) /
1136             csum_size) {
1137                 int extend_nr;
1138                 u64 tmp;
1139                 u32 diff;
1140
1141                 tmp = sums->len - total_bytes;
1142                 tmp >>= fs_info->sectorsize_bits;
1143                 WARN_ON(tmp < 1);
1144                 extend_nr = max_t(int, 1, tmp);
1145
1146                 /*
1147                  * A log tree can already have checksum items with a subset of
1148                  * the checksums we are trying to log. This can happen after
1149                  * doing a sequence of partial writes into prealloc extents and
1150                  * fsyncs in between, with a full fsync logging a larger subrange
1151                  * of an extent for which a previous fast fsync logged a smaller
1152                  * subrange. And this happens in particular due to merging file
1153                  * extent items when we complete an ordered extent for a range
1154                  * covered by a prealloc extent - this is done at
1155                  * btrfs_mark_extent_written().
1156                  *
1157                  * So if we try to extend the previous checksum item, which has
1158                  * a range that ends at the start of the range we want to insert,
1159                  * make sure we don't extend beyond the start offset of the next
1160                  * checksum item. If we are at the last item in the leaf, then
1161                  * forget the optimization of extending and add a new checksum
1162                  * item - it is not worth the complexity of releasing the path,
1163                  * getting the first key for the next leaf, repeat the btree
1164                  * search, etc, because log trees are temporary anyway and it
1165                  * would only save a few bytes of leaf space.
1166                  */
1167                 if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
1168                         if (path->slots[0] + 1 >=
1169                             btrfs_header_nritems(path->nodes[0])) {
1170                                 ret = find_next_csum_offset(root, path, &next_offset);
1171                                 if (ret < 0)
1172                                         goto out;
1173                                 found_next = 1;
1174                                 goto insert;
1175                         }
1176
1177                         ret = find_next_csum_offset(root, path, &next_offset);
1178                         if (ret < 0)
1179                                 goto out;
1180
1181                         tmp = (next_offset - bytenr) >> fs_info->sectorsize_bits;
1182                         if (tmp <= INT_MAX)
1183                                 extend_nr = min_t(int, extend_nr, tmp);
1184                 }
1185
1186                 diff = (csum_offset + extend_nr) * csum_size;
1187                 diff = min(diff,
1188                            MAX_CSUM_ITEMS(fs_info, csum_size) * csum_size);
1189
1190                 diff = diff - btrfs_item_size(leaf, path->slots[0]);
1191                 diff = min_t(u32, btrfs_leaf_free_space(leaf), diff);
1192                 diff /= csum_size;
1193                 diff *= csum_size;
1194
1195                 btrfs_extend_item(path, diff);
1196                 ret = 0;
1197                 goto csum;
1198         }
1199
1200 insert:
1201         btrfs_release_path(path);
1202         csum_offset = 0;
1203         if (found_next) {
1204                 u64 tmp;
1205
1206                 tmp = sums->len - total_bytes;
1207                 tmp >>= fs_info->sectorsize_bits;
1208                 tmp = min(tmp, (next_offset - file_key.offset) >>
1209                                          fs_info->sectorsize_bits);
1210
1211                 tmp = max_t(u64, 1, tmp);
1212                 tmp = min_t(u64, tmp, MAX_CSUM_ITEMS(fs_info, csum_size));
1213                 ins_size = csum_size * tmp;
1214         } else {
1215                 ins_size = csum_size;
1216         }
1217         ret = btrfs_insert_empty_item(trans, root, path, &file_key,
1218                                       ins_size);
1219         if (ret < 0)
1220                 goto out;
1221         if (WARN_ON(ret != 0))
1222                 goto out;
1223         leaf = path->nodes[0];
1224 csum:
1225         item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item);
1226         item_end = (struct btrfs_csum_item *)((unsigned char *)item +
1227                                       btrfs_item_size(leaf, path->slots[0]));
1228         item = (struct btrfs_csum_item *)((unsigned char *)item +
1229                                           csum_offset * csum_size);
1230 found:
1231         ins_size = (u32)(sums->len - total_bytes) >> fs_info->sectorsize_bits;
1232         ins_size *= csum_size;
1233         ins_size = min_t(u32, (unsigned long)item_end - (unsigned long)item,
1234                               ins_size);
1235         write_extent_buffer(leaf, sums->sums + index, (unsigned long)item,
1236                             ins_size);
1237
1238         index += ins_size;
1239         ins_size /= csum_size;
1240         total_bytes += ins_size * fs_info->sectorsize;
1241
1242         btrfs_mark_buffer_dirty(path->nodes[0]);
1243         if (total_bytes < sums->len) {
1244                 btrfs_release_path(path);
1245                 cond_resched();
1246                 goto again;
1247         }
1248 out:
1249         btrfs_free_path(path);
1250         return ret;
1251 }
1252
1253 void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode,
1254                                      const struct btrfs_path *path,
1255                                      struct btrfs_file_extent_item *fi,
1256                                      struct extent_map *em)
1257 {
1258         struct btrfs_fs_info *fs_info = inode->root->fs_info;
1259         struct btrfs_root *root = inode->root;
1260         struct extent_buffer *leaf = path->nodes[0];
1261         const int slot = path->slots[0];
1262         struct btrfs_key key;
1263         u64 extent_start, extent_end;
1264         u64 bytenr;
1265         u8 type = btrfs_file_extent_type(leaf, fi);
1266         int compress_type = btrfs_file_extent_compression(leaf, fi);
1267
1268         btrfs_item_key_to_cpu(leaf, &key, slot);
1269         extent_start = key.offset;
1270         extent_end = btrfs_file_extent_end(path);
1271         em->ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
1272         em->generation = btrfs_file_extent_generation(leaf, fi);
1273         if (type == BTRFS_FILE_EXTENT_REG ||
1274             type == BTRFS_FILE_EXTENT_PREALLOC) {
1275                 em->start = extent_start;
1276                 em->len = extent_end - extent_start;
1277                 em->orig_start = extent_start -
1278                         btrfs_file_extent_offset(leaf, fi);
1279                 em->orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
1280                 bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1281                 if (bytenr == 0) {
1282                         em->block_start = EXTENT_MAP_HOLE;
1283                         return;
1284                 }
1285                 if (compress_type != BTRFS_COMPRESS_NONE) {
1286                         set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
1287                         em->compress_type = compress_type;
1288                         em->block_start = bytenr;
1289                         em->block_len = em->orig_block_len;
1290                 } else {
1291                         bytenr += btrfs_file_extent_offset(leaf, fi);
1292                         em->block_start = bytenr;
1293                         em->block_len = em->len;
1294                         if (type == BTRFS_FILE_EXTENT_PREALLOC)
1295                                 set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
1296                 }
1297         } else if (type == BTRFS_FILE_EXTENT_INLINE) {
1298                 em->block_start = EXTENT_MAP_INLINE;
1299                 em->start = extent_start;
1300                 em->len = extent_end - extent_start;
1301                 /*
1302                  * Initialize orig_start and block_len with the same values
1303                  * as in inode.c:btrfs_get_extent().
1304                  */
1305                 em->orig_start = EXTENT_MAP_HOLE;
1306                 em->block_len = (u64)-1;
1307                 em->compress_type = compress_type;
1308                 if (compress_type != BTRFS_COMPRESS_NONE)
1309                         set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
1310         } else {
1311                 btrfs_err(fs_info,
1312                           "unknown file extent item type %d, inode %llu, offset %llu, "
1313                           "root %llu", type, btrfs_ino(inode), extent_start,
1314                           root->root_key.objectid);
1315         }
1316 }
1317
1318 /*
1319  * Returns the end offset (non inclusive) of the file extent item the given path
1320  * points to. If it points to an inline extent, the returned offset is rounded
1321  * up to the sector size.
1322  */
1323 u64 btrfs_file_extent_end(const struct btrfs_path *path)
1324 {
1325         const struct extent_buffer *leaf = path->nodes[0];
1326         const int slot = path->slots[0];
1327         struct btrfs_file_extent_item *fi;
1328         struct btrfs_key key;
1329         u64 end;
1330
1331         btrfs_item_key_to_cpu(leaf, &key, slot);
1332         ASSERT(key.type == BTRFS_EXTENT_DATA_KEY);
1333         fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
1334
1335         if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE) {
1336                 end = btrfs_file_extent_ram_bytes(leaf, fi);
1337                 end = ALIGN(key.offset + end, leaf->fs_info->sectorsize);
1338         } else {
1339                 end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
1340         }
1341
1342         return end;
1343 }